pulumi/pkg/engine/journal.go

244 lines
7.1 KiB
Go
Raw Normal View History

// Copyright 2016-2022, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package engine
import (
"errors"
"github.com/pulumi/pulumi/pkg/v3/resource/deploy"
"github.com/pulumi/pulumi/pkg/v3/secrets"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
)
var _ = SnapshotManager((*Journal)(nil))
type JournalEntryKind int
const (
JournalEntryBegin JournalEntryKind = 0
JournalEntrySuccess JournalEntryKind = 1
JournalEntryFailure JournalEntryKind = 2
JournalEntryOutputs JournalEntryKind = 4
)
type JournalEntry struct {
Kind JournalEntryKind
Step deploy.Step
}
type JournalEntries []JournalEntry
[engine] Only record a resource's chosen alias. (#9288) As we discovered when removing aliases from the state entirely, the snapshotter needs to be alias-aware so that it can fix up references to resources that were aliased. After a resource operation finishes, the snapshotter needs to write out a new copy of the snapshot. However, at the time we write the snapshot, there may be resources that have not yet been registered that refer to the just-registered resources by a different URN due to aliasing. Those references need to be fixed up prior to writing the snapshot in order to preserve the snapshot's integrity (in particular, the property that all URNs refer to resources that exist in the snapshot). For example, consider the following simple dependency graph: A <-- B. When that graph is serialized, B will contain a reference to A in its dependency list. Let the next run of the program produces the graph A' <-- B where A' is aliased to A. After A' is registered, the snapshotter needs to write a snapshot that contains its state, but B must also be updated so it references A' instead of A, which will no longer be in the snapshot. These changes take advantage of the fact that although a resource can provide multiple aliases, it can only ever resolve those aliases to a single resource in the existing state. Therefore, at the time the statefile is fixed up, each resource in the statefile could only have been aliased to a single old resource, and it is sufficient to store only the URN of the chosen resource rather than all possible aliases. In addition to preserving the ability to fix up references to aliased resources, retaining the chosen alias allows the history of a logical resource to be followed across aliases.
2022-03-28 15:36:08 +00:00
func (entries JournalEntries) Snap(base *deploy.Snapshot) (*deploy.Snapshot, error) {
// Build up a list of current resources by replaying the journal.
resources, dones := []*resource.State{}, make(map[*resource.State]bool)
ops, doneOps := []resource.Operation{}, make(map[*resource.State]bool)
for _, e := range entries {
logging.V(7).Infof("%v %v (%v)", e.Step.Op(), e.Step.URN(), e.Kind)
// Begin journal entries add pending operations to the snapshot. As we see success or failure
// entries, we'll record them in doneOps.
switch e.Kind {
case JournalEntryBegin:
switch e.Step.Op() {
case deploy.OpCreate, deploy.OpCreateReplacement:
ops = append(ops, resource.NewOperation(e.Step.New(), resource.OperationTypeCreating))
case deploy.OpDelete, deploy.OpDeleteReplaced, deploy.OpReadDiscard, deploy.OpDiscardReplaced:
ops = append(ops, resource.NewOperation(e.Step.Old(), resource.OperationTypeDeleting))
case deploy.OpRead, deploy.OpReadReplacement:
ops = append(ops, resource.NewOperation(e.Step.New(), resource.OperationTypeReading))
case deploy.OpUpdate:
ops = append(ops, resource.NewOperation(e.Step.New(), resource.OperationTypeUpdating))
case deploy.OpImport, deploy.OpImportReplacement:
ops = append(ops, resource.NewOperation(e.Step.New(), resource.OperationTypeImporting))
}
case JournalEntryFailure, JournalEntrySuccess:
switch e.Step.Op() {
sdk/go: Remove 'nolint' directives from package docs Go treats comments that match the following regex as directives. //[a-z0-9]+:[a-z0-9] Comments that are directives don't show in an entity's documentation. https://github.com/golang/go/commit/5a550b695117f07a4f2454039a4871250cd3ed09#diff-f56160fd9fcea272966a8a1d692ad9f49206fdd8dbcbfe384865a98cd9bc2749R165 Our code has `//nolint` directives that now show in the API Reference. This is because these directives are in one of the following forms, which don't get this special treatment. // nolint:foo //nolint: foo This change fixes all such directives found by the regex: `// nolint|//nolint: `. See bottom of commit for command used for the fix. Verification: Here's the output of `go doc` on some entities before and after this change. Before ``` % go doc github.com/pulumi/pulumi/sdk/v3/go/pulumi | head -n8 package pulumi // import "github.com/pulumi/pulumi/sdk/v3/go/pulumi" nolint: lll, interfacer nolint: lll, interfacer const EnvOrganization = "PULUMI_ORGANIZATION" ... var ErrPlugins = errors.New("pulumi: plugins requested") ``` After ``` % go doc github.com/pulumi/pulumi/sdk/v3/go/pulumi | head -n8 package pulumi // import "github.com/pulumi/pulumi/sdk/v3/go/pulumi" const EnvOrganization = "PULUMI_ORGANIZATION" ... var ErrPlugins = errors.New("pulumi: plugins requested") func BoolRef(v bool) *bool func Float64Ref(v float64) *float64 func IntRef(v int) *int func IsSecret(o Output) bool ``` Before ``` % go doc github.com/pulumi/pulumi/sdk/v3/go/pulumi URN_ package pulumi // import "github.com/pulumi/pulumi/sdk/v3/go/pulumi" func URN_(o string) ResourceOption URN_ is an optional URN of a previously-registered resource of this type to read from the engine. nolint: revive ``` After: ``` % go doc github.com/pulumi/pulumi/sdk/v3/go/pulumi URN_ package pulumi // import "github.com/pulumi/pulumi/sdk/v3/go/pulumi" func URN_(o string) ResourceOption URN_ is an optional URN of a previously-registered resource of this type to read from the engine. ``` Note that golangci-lint offers a 'nolintlint' linter that finds such miuses of nolint, but it also finds other issues so I've deferred that to a follow up PR. Resolves #11785 Related: https://github.com/golangci/golangci-lint/issues/892 [git-generate] FILES=$(mktemp) rg -l '// nolint|//nolint: ' | tee "$FILES" | xargs perl -p -i -e ' s|// nolint|//nolint|g; s|//nolint: |//nolint:|g; ' rg '.go$' < "$FILES" | xargs gofmt -w -s
2023-01-06 00:07:45 +00:00
//nolint:lll
case deploy.OpCreate, deploy.OpCreateReplacement, deploy.OpRead, deploy.OpReadReplacement, deploy.OpUpdate,
deploy.OpImport, deploy.OpImportReplacement:
doneOps[e.Step.New()] = true
case deploy.OpDelete, deploy.OpDeleteReplaced, deploy.OpReadDiscard, deploy.OpDiscardReplaced:
doneOps[e.Step.Old()] = true
}
turn on the golangci-lint exhaustive linter (#15028) Turn on the golangci-lint exhaustive linter. This is the first step towards catching more missing cases during development rather than in tests, or in production. This might be best reviewed commit-by-commit, as the first commit turns on the linter with the `default-signifies-exhaustive: true` option set, which requires a lot less changes in the current codebase. I think it's probably worth doing the second commit as well, as that will get us the real benefits, even though we end up with a little bit more churn. However it means all the `switch` statements are covered, which isn't the case after the first commit, since we do have a lot of `default` statements that just call `assert.Fail`. Fixes #14601 ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-01-17 16:50:41 +00:00
case JournalEntryOutputs:
// We do nothing for outputs, since they don't affect the snapshot.
}
// Now mark resources done as necessary.
if e.Kind == JournalEntrySuccess {
switch e.Step.Op() {
case deploy.OpSame:
step, ok := e.Step.(*deploy.SameStep)
contract.Assertf(ok, "expected *deploy.SameStep, got %T", e.Step)
if !step.IsSkippedCreate() {
resources = append(resources, e.Step.New())
dones[e.Step.Old()] = true
}
case deploy.OpUpdate:
resources = append(resources, e.Step.New())
dones[e.Step.Old()] = true
case deploy.OpCreate, deploy.OpCreateReplacement:
resources = append(resources, e.Step.New())
if old := e.Step.Old(); old != nil && old.PendingReplacement {
dones[old] = true
}
case deploy.OpDelete, deploy.OpDeleteReplaced, deploy.OpReadDiscard, deploy.OpDiscardReplaced:
if old := e.Step.Old(); !old.PendingReplacement {
dones[old] = true
}
case deploy.OpReplace:
// do nothing.
case deploy.OpRead, deploy.OpReadReplacement:
resources = append(resources, e.Step.New())
if e.Step.Old() != nil {
dones[e.Step.Old()] = true
}
case deploy.OpRemovePendingReplace:
dones[e.Step.Old()] = true
case deploy.OpImport, deploy.OpImportReplacement:
resources = append(resources, e.Step.New())
dones[e.Step.New()] = true
}
}
}
// Append any resources from the base snapshot that were not produced by the current snapshot.
// See backend.SnapshotManager.snap for why this works.
if base != nil {
for _, res := range base.Resources {
if !dones[res] {
resources = append(resources, res)
}
}
}
// Append any pending operations.
var operations []resource.Operation
for _, op := range ops {
if !doneOps[op.Resource] {
operations = append(operations, op)
}
}
if base != nil {
// Track pending create operations from the base snapshot
// and propagate them to the new snapshot: we don't want to clear pending CREATE operations
// because these must require user intervention to be cleared or resolved.
for _, pendingOperation := range base.PendingOperations {
if pendingOperation.Type == resource.OperationTypeCreating {
operations = append(operations, pendingOperation)
}
}
}
// If we have a base snapshot, copy over its secrets manager.
var secretsManager secrets.Manager
if base != nil {
secretsManager = base.SecretsManager
}
manifest := deploy.Manifest{}
manifest.Magic = manifest.NewMagic()
[engine] Only record a resource's chosen alias. (#9288) As we discovered when removing aliases from the state entirely, the snapshotter needs to be alias-aware so that it can fix up references to resources that were aliased. After a resource operation finishes, the snapshotter needs to write out a new copy of the snapshot. However, at the time we write the snapshot, there may be resources that have not yet been registered that refer to the just-registered resources by a different URN due to aliasing. Those references need to be fixed up prior to writing the snapshot in order to preserve the snapshot's integrity (in particular, the property that all URNs refer to resources that exist in the snapshot). For example, consider the following simple dependency graph: A <-- B. When that graph is serialized, B will contain a reference to A in its dependency list. Let the next run of the program produces the graph A' <-- B where A' is aliased to A. After A' is registered, the snapshotter needs to write a snapshot that contains its state, but B must also be updated so it references A' instead of A, which will no longer be in the snapshot. These changes take advantage of the fact that although a resource can provide multiple aliases, it can only ever resolve those aliases to a single resource in the existing state. Therefore, at the time the statefile is fixed up, each resource in the statefile could only have been aliased to a single old resource, and it is sufficient to store only the URN of the chosen resource rather than all possible aliases. In addition to preserving the ability to fix up references to aliased resources, retaining the chosen alias allows the history of a logical resource to be followed across aliases.
2022-03-28 15:36:08 +00:00
snap := deploy.NewSnapshot(manifest, secretsManager, resources, operations)
normSnap, err := snap.NormalizeURNReferences()
[engine] Only record a resource's chosen alias. (#9288) As we discovered when removing aliases from the state entirely, the snapshotter needs to be alias-aware so that it can fix up references to resources that were aliased. After a resource operation finishes, the snapshotter needs to write out a new copy of the snapshot. However, at the time we write the snapshot, there may be resources that have not yet been registered that refer to the just-registered resources by a different URN due to aliasing. Those references need to be fixed up prior to writing the snapshot in order to preserve the snapshot's integrity (in particular, the property that all URNs refer to resources that exist in the snapshot). For example, consider the following simple dependency graph: A <-- B. When that graph is serialized, B will contain a reference to A in its dependency list. Let the next run of the program produces the graph A' <-- B where A' is aliased to A. After A' is registered, the snapshotter needs to write a snapshot that contains its state, but B must also be updated so it references A' instead of A, which will no longer be in the snapshot. These changes take advantage of the fact that although a resource can provide multiple aliases, it can only ever resolve those aliases to a single resource in the existing state. Therefore, at the time the statefile is fixed up, each resource in the statefile could only have been aliased to a single old resource, and it is sufficient to store only the URN of the chosen resource rather than all possible aliases. In addition to preserving the ability to fix up references to aliased resources, retaining the chosen alias allows the history of a logical resource to be followed across aliases.
2022-03-28 15:36:08 +00:00
if err != nil {
return snap, err
}
return normSnap, normSnap.VerifyIntegrity()
}
type Journal struct {
entries JournalEntries
events chan JournalEntry
cancel chan bool
done chan bool
}
func (j *Journal) Entries() JournalEntries {
<-j.done
return j.entries
}
func (j *Journal) Close() error {
close(j.cancel)
<-j.done
return nil
}
func (j *Journal) BeginMutation(step deploy.Step) (SnapshotMutation, error) {
select {
case j.events <- JournalEntry{Kind: JournalEntryBegin, Step: step}:
return j, nil
case <-j.cancel:
return nil, errors.New("journal closed")
}
}
func (j *Journal) End(step deploy.Step, success bool) error {
kind := JournalEntryFailure
if success {
kind = JournalEntrySuccess
}
select {
case j.events <- JournalEntry{Kind: kind, Step: step}:
return nil
case <-j.cancel:
return errors.New("journal closed")
}
}
func (j *Journal) RegisterResourceOutputs(step deploy.Step) error {
select {
case j.events <- JournalEntry{Kind: JournalEntryOutputs, Step: step}:
return nil
case <-j.cancel:
return errors.New("journal closed")
}
}
func (j *Journal) RecordPlugin(plugin workspace.PluginInfo) error {
return nil
}
[engine] Only record a resource's chosen alias. (#9288) As we discovered when removing aliases from the state entirely, the snapshotter needs to be alias-aware so that it can fix up references to resources that were aliased. After a resource operation finishes, the snapshotter needs to write out a new copy of the snapshot. However, at the time we write the snapshot, there may be resources that have not yet been registered that refer to the just-registered resources by a different URN due to aliasing. Those references need to be fixed up prior to writing the snapshot in order to preserve the snapshot's integrity (in particular, the property that all URNs refer to resources that exist in the snapshot). For example, consider the following simple dependency graph: A <-- B. When that graph is serialized, B will contain a reference to A in its dependency list. Let the next run of the program produces the graph A' <-- B where A' is aliased to A. After A' is registered, the snapshotter needs to write a snapshot that contains its state, but B must also be updated so it references A' instead of A, which will no longer be in the snapshot. These changes take advantage of the fact that although a resource can provide multiple aliases, it can only ever resolve those aliases to a single resource in the existing state. Therefore, at the time the statefile is fixed up, each resource in the statefile could only have been aliased to a single old resource, and it is sufficient to store only the URN of the chosen resource rather than all possible aliases. In addition to preserving the ability to fix up references to aliased resources, retaining the chosen alias allows the history of a logical resource to be followed across aliases.
2022-03-28 15:36:08 +00:00
func (j *Journal) Snap(base *deploy.Snapshot) (*deploy.Snapshot, error) {
return j.entries.Snap(base)
}
func NewJournal() *Journal {
j := &Journal{
events: make(chan JournalEntry),
cancel: make(chan bool),
done: make(chan bool),
}
go func() {
for {
select {
case <-j.cancel:
close(j.done)
return
case e := <-j.events:
j.entries = append(j.entries, e)
}
}
}()
return j
}