2022-07-18 18:04:19 +00:00
|
|
|
// Copyright 2016-2022, Pulumi Corporation.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2020-10-15 17:35:09 +00:00
|
|
|
package engine
|
|
|
|
|
|
|
|
import (
|
2021-11-13 02:37:17 +00:00
|
|
|
"errors"
|
2020-10-15 17:35:09 +00:00
|
|
|
|
2021-03-17 13:20:05 +00:00
|
|
|
"github.com/pulumi/pulumi/pkg/v3/resource/deploy"
|
|
|
|
"github.com/pulumi/pulumi/pkg/v3/secrets"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
|
2023-07-21 15:06:15 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
|
2021-03-17 13:20:05 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
|
2020-10-15 17:35:09 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
var _ = SnapshotManager((*Journal)(nil))
|
|
|
|
|
|
|
|
type JournalEntryKind int
|
|
|
|
|
|
|
|
const (
|
|
|
|
JournalEntryBegin JournalEntryKind = 0
|
|
|
|
JournalEntrySuccess JournalEntryKind = 1
|
|
|
|
JournalEntryFailure JournalEntryKind = 2
|
|
|
|
JournalEntryOutputs JournalEntryKind = 4
|
|
|
|
)
|
|
|
|
|
|
|
|
type JournalEntry struct {
|
|
|
|
Kind JournalEntryKind
|
|
|
|
Step deploy.Step
|
|
|
|
}
|
|
|
|
|
|
|
|
type JournalEntries []JournalEntry
|
|
|
|
|
[engine] Only record a resource's chosen alias. (#9288)
As we discovered when removing aliases from the state entirely, the
snapshotter needs to be alias-aware so that it can fix up references to
resources that were aliased. After a resource operation finishes, the
snapshotter needs to write out a new copy of the snapshot. However, at
the time we write the snapshot, there may be resources that have not yet
been registered that refer to the just-registered resources by a
different URN due to aliasing. Those references need to be fixed up
prior to writing the snapshot in order to preserve the snapshot's
integrity (in particular, the property that all URNs refer to resources
that exist in the snapshot).
For example, consider the following simple dependency graph: A <-- B.
When that graph is serialized, B will contain a reference to A in its
dependency list. Let the next run of the program produces the graph A'
<-- B where A' is aliased to A. After A' is registered, the snapshotter
needs to write a snapshot that contains its state, but B must also be
updated so it references A' instead of A, which will no longer be in the
snapshot.
These changes take advantage of the fact that although a resource can
provide multiple aliases, it can only ever resolve those aliases to a
single resource in the existing state. Therefore, at the time the
statefile is fixed up, each resource in the statefile could only have
been aliased to a single old resource, and it is sufficient to store
only the URN of the chosen resource rather than all possible aliases. In
addition to preserving the ability to fix up references to aliased
resources, retaining the chosen alias allows the history of a logical
resource to be followed across aliases.
2022-03-28 15:36:08 +00:00
|
|
|
func (entries JournalEntries) Snap(base *deploy.Snapshot) (*deploy.Snapshot, error) {
|
2020-10-15 17:35:09 +00:00
|
|
|
// Build up a list of current resources by replaying the journal.
|
|
|
|
resources, dones := []*resource.State{}, make(map[*resource.State]bool)
|
|
|
|
ops, doneOps := []resource.Operation{}, make(map[*resource.State]bool)
|
|
|
|
for _, e := range entries {
|
|
|
|
logging.V(7).Infof("%v %v (%v)", e.Step.Op(), e.Step.URN(), e.Kind)
|
|
|
|
|
|
|
|
// Begin journal entries add pending operations to the snapshot. As we see success or failure
|
|
|
|
// entries, we'll record them in doneOps.
|
|
|
|
switch e.Kind {
|
|
|
|
case JournalEntryBegin:
|
|
|
|
switch e.Step.Op() {
|
|
|
|
case deploy.OpCreate, deploy.OpCreateReplacement:
|
|
|
|
ops = append(ops, resource.NewOperation(e.Step.New(), resource.OperationTypeCreating))
|
|
|
|
case deploy.OpDelete, deploy.OpDeleteReplaced, deploy.OpReadDiscard, deploy.OpDiscardReplaced:
|
|
|
|
ops = append(ops, resource.NewOperation(e.Step.Old(), resource.OperationTypeDeleting))
|
|
|
|
case deploy.OpRead, deploy.OpReadReplacement:
|
|
|
|
ops = append(ops, resource.NewOperation(e.Step.New(), resource.OperationTypeReading))
|
|
|
|
case deploy.OpUpdate:
|
|
|
|
ops = append(ops, resource.NewOperation(e.Step.New(), resource.OperationTypeUpdating))
|
|
|
|
case deploy.OpImport, deploy.OpImportReplacement:
|
|
|
|
ops = append(ops, resource.NewOperation(e.Step.New(), resource.OperationTypeImporting))
|
|
|
|
}
|
|
|
|
case JournalEntryFailure, JournalEntrySuccess:
|
|
|
|
switch e.Step.Op() {
|
2023-01-06 00:07:45 +00:00
|
|
|
//nolint:lll
|
2020-10-15 17:35:09 +00:00
|
|
|
case deploy.OpCreate, deploy.OpCreateReplacement, deploy.OpRead, deploy.OpReadReplacement, deploy.OpUpdate,
|
|
|
|
deploy.OpImport, deploy.OpImportReplacement:
|
|
|
|
doneOps[e.Step.New()] = true
|
|
|
|
case deploy.OpDelete, deploy.OpDeleteReplaced, deploy.OpReadDiscard, deploy.OpDiscardReplaced:
|
|
|
|
doneOps[e.Step.Old()] = true
|
|
|
|
}
|
turn on the golangci-lint exhaustive linter (#15028)
Turn on the golangci-lint exhaustive linter. This is the first step
towards catching more missing cases during development rather than
in tests, or in production.
This might be best reviewed commit-by-commit, as the first commit turns
on the linter with the `default-signifies-exhaustive: true` option set,
which requires a lot less changes in the current codebase.
I think it's probably worth doing the second commit as well, as that
will get us the real benefits, even though we end up with a little bit
more churn. However it means all the `switch` statements are covered,
which isn't the case after the first commit, since we do have a lot of
`default` statements that just call `assert.Fail`.
Fixes #14601
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [x] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [ ] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2024-01-17 16:50:41 +00:00
|
|
|
case JournalEntryOutputs:
|
|
|
|
// We do nothing for outputs, since they don't affect the snapshot.
|
2020-10-15 17:35:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Now mark resources done as necessary.
|
|
|
|
if e.Kind == JournalEntrySuccess {
|
|
|
|
switch e.Step.Op() {
|
2023-07-21 15:06:15 +00:00
|
|
|
case deploy.OpSame:
|
|
|
|
step, ok := e.Step.(*deploy.SameStep)
|
|
|
|
contract.Assertf(ok, "expected *deploy.SameStep, got %T", e.Step)
|
|
|
|
if !step.IsSkippedCreate() {
|
|
|
|
resources = append(resources, e.Step.New())
|
|
|
|
dones[e.Step.Old()] = true
|
|
|
|
}
|
|
|
|
case deploy.OpUpdate:
|
2020-10-15 17:35:09 +00:00
|
|
|
resources = append(resources, e.Step.New())
|
|
|
|
dones[e.Step.Old()] = true
|
|
|
|
case deploy.OpCreate, deploy.OpCreateReplacement:
|
|
|
|
resources = append(resources, e.Step.New())
|
|
|
|
if old := e.Step.Old(); old != nil && old.PendingReplacement {
|
|
|
|
dones[old] = true
|
|
|
|
}
|
|
|
|
case deploy.OpDelete, deploy.OpDeleteReplaced, deploy.OpReadDiscard, deploy.OpDiscardReplaced:
|
|
|
|
if old := e.Step.Old(); !old.PendingReplacement {
|
|
|
|
dones[old] = true
|
|
|
|
}
|
|
|
|
case deploy.OpReplace:
|
|
|
|
// do nothing.
|
|
|
|
case deploy.OpRead, deploy.OpReadReplacement:
|
|
|
|
resources = append(resources, e.Step.New())
|
|
|
|
if e.Step.Old() != nil {
|
|
|
|
dones[e.Step.Old()] = true
|
|
|
|
}
|
|
|
|
case deploy.OpRemovePendingReplace:
|
|
|
|
dones[e.Step.Old()] = true
|
|
|
|
case deploy.OpImport, deploy.OpImportReplacement:
|
|
|
|
resources = append(resources, e.Step.New())
|
|
|
|
dones[e.Step.New()] = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Append any resources from the base snapshot that were not produced by the current snapshot.
|
|
|
|
// See backend.SnapshotManager.snap for why this works.
|
|
|
|
if base != nil {
|
|
|
|
for _, res := range base.Resources {
|
|
|
|
if !dones[res] {
|
|
|
|
resources = append(resources, res)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Append any pending operations.
|
|
|
|
var operations []resource.Operation
|
|
|
|
for _, op := range ops {
|
|
|
|
if !doneOps[op.Resource] {
|
|
|
|
operations = append(operations, op)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-25 09:59:19 +00:00
|
|
|
if base != nil {
|
|
|
|
// Track pending create operations from the base snapshot
|
|
|
|
// and propagate them to the new snapshot: we don't want to clear pending CREATE operations
|
|
|
|
// because these must require user intervention to be cleared or resolved.
|
|
|
|
for _, pendingOperation := range base.PendingOperations {
|
|
|
|
if pendingOperation.Type == resource.OperationTypeCreating {
|
|
|
|
operations = append(operations, pendingOperation)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-10-15 17:35:09 +00:00
|
|
|
// If we have a base snapshot, copy over its secrets manager.
|
|
|
|
var secretsManager secrets.Manager
|
|
|
|
if base != nil {
|
|
|
|
secretsManager = base.SecretsManager
|
|
|
|
}
|
|
|
|
|
|
|
|
manifest := deploy.Manifest{}
|
|
|
|
manifest.Magic = manifest.NewMagic()
|
|
|
|
|
[engine] Only record a resource's chosen alias. (#9288)
As we discovered when removing aliases from the state entirely, the
snapshotter needs to be alias-aware so that it can fix up references to
resources that were aliased. After a resource operation finishes, the
snapshotter needs to write out a new copy of the snapshot. However, at
the time we write the snapshot, there may be resources that have not yet
been registered that refer to the just-registered resources by a
different URN due to aliasing. Those references need to be fixed up
prior to writing the snapshot in order to preserve the snapshot's
integrity (in particular, the property that all URNs refer to resources
that exist in the snapshot).
For example, consider the following simple dependency graph: A <-- B.
When that graph is serialized, B will contain a reference to A in its
dependency list. Let the next run of the program produces the graph A'
<-- B where A' is aliased to A. After A' is registered, the snapshotter
needs to write a snapshot that contains its state, but B must also be
updated so it references A' instead of A, which will no longer be in the
snapshot.
These changes take advantage of the fact that although a resource can
provide multiple aliases, it can only ever resolve those aliases to a
single resource in the existing state. Therefore, at the time the
statefile is fixed up, each resource in the statefile could only have
been aliased to a single old resource, and it is sufficient to store
only the URN of the chosen resource rather than all possible aliases. In
addition to preserving the ability to fix up references to aliased
resources, retaining the chosen alias allows the history of a logical
resource to be followed across aliases.
2022-03-28 15:36:08 +00:00
|
|
|
snap := deploy.NewSnapshot(manifest, secretsManager, resources, operations)
|
2022-07-18 18:04:19 +00:00
|
|
|
normSnap, err := snap.NormalizeURNReferences()
|
[engine] Only record a resource's chosen alias. (#9288)
As we discovered when removing aliases from the state entirely, the
snapshotter needs to be alias-aware so that it can fix up references to
resources that were aliased. After a resource operation finishes, the
snapshotter needs to write out a new copy of the snapshot. However, at
the time we write the snapshot, there may be resources that have not yet
been registered that refer to the just-registered resources by a
different URN due to aliasing. Those references need to be fixed up
prior to writing the snapshot in order to preserve the snapshot's
integrity (in particular, the property that all URNs refer to resources
that exist in the snapshot).
For example, consider the following simple dependency graph: A <-- B.
When that graph is serialized, B will contain a reference to A in its
dependency list. Let the next run of the program produces the graph A'
<-- B where A' is aliased to A. After A' is registered, the snapshotter
needs to write a snapshot that contains its state, but B must also be
updated so it references A' instead of A, which will no longer be in the
snapshot.
These changes take advantage of the fact that although a resource can
provide multiple aliases, it can only ever resolve those aliases to a
single resource in the existing state. Therefore, at the time the
statefile is fixed up, each resource in the statefile could only have
been aliased to a single old resource, and it is sufficient to store
only the URN of the chosen resource rather than all possible aliases. In
addition to preserving the ability to fix up references to aliased
resources, retaining the chosen alias allows the history of a logical
resource to be followed across aliases.
2022-03-28 15:36:08 +00:00
|
|
|
if err != nil {
|
|
|
|
return snap, err
|
|
|
|
}
|
2022-07-18 18:04:19 +00:00
|
|
|
return normSnap, normSnap.VerifyIntegrity()
|
2020-10-15 17:35:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type Journal struct {
|
|
|
|
entries JournalEntries
|
|
|
|
events chan JournalEntry
|
|
|
|
cancel chan bool
|
|
|
|
done chan bool
|
|
|
|
}
|
|
|
|
|
2024-01-03 17:32:13 +00:00
|
|
|
func (j *Journal) Entries() JournalEntries {
|
2020-10-15 17:35:09 +00:00
|
|
|
<-j.done
|
|
|
|
|
|
|
|
return j.entries
|
|
|
|
}
|
|
|
|
|
|
|
|
func (j *Journal) Close() error {
|
|
|
|
close(j.cancel)
|
|
|
|
<-j.done
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (j *Journal) BeginMutation(step deploy.Step) (SnapshotMutation, error) {
|
|
|
|
select {
|
|
|
|
case j.events <- JournalEntry{Kind: JournalEntryBegin, Step: step}:
|
|
|
|
return j, nil
|
|
|
|
case <-j.cancel:
|
|
|
|
return nil, errors.New("journal closed")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (j *Journal) End(step deploy.Step, success bool) error {
|
|
|
|
kind := JournalEntryFailure
|
|
|
|
if success {
|
|
|
|
kind = JournalEntrySuccess
|
|
|
|
}
|
|
|
|
select {
|
|
|
|
case j.events <- JournalEntry{Kind: kind, Step: step}:
|
|
|
|
return nil
|
|
|
|
case <-j.cancel:
|
|
|
|
return errors.New("journal closed")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (j *Journal) RegisterResourceOutputs(step deploy.Step) error {
|
|
|
|
select {
|
|
|
|
case j.events <- JournalEntry{Kind: JournalEntryOutputs, Step: step}:
|
|
|
|
return nil
|
|
|
|
case <-j.cancel:
|
|
|
|
return errors.New("journal closed")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (j *Journal) RecordPlugin(plugin workspace.PluginInfo) error {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
[engine] Only record a resource's chosen alias. (#9288)
As we discovered when removing aliases from the state entirely, the
snapshotter needs to be alias-aware so that it can fix up references to
resources that were aliased. After a resource operation finishes, the
snapshotter needs to write out a new copy of the snapshot. However, at
the time we write the snapshot, there may be resources that have not yet
been registered that refer to the just-registered resources by a
different URN due to aliasing. Those references need to be fixed up
prior to writing the snapshot in order to preserve the snapshot's
integrity (in particular, the property that all URNs refer to resources
that exist in the snapshot).
For example, consider the following simple dependency graph: A <-- B.
When that graph is serialized, B will contain a reference to A in its
dependency list. Let the next run of the program produces the graph A'
<-- B where A' is aliased to A. After A' is registered, the snapshotter
needs to write a snapshot that contains its state, but B must also be
updated so it references A' instead of A, which will no longer be in the
snapshot.
These changes take advantage of the fact that although a resource can
provide multiple aliases, it can only ever resolve those aliases to a
single resource in the existing state. Therefore, at the time the
statefile is fixed up, each resource in the statefile could only have
been aliased to a single old resource, and it is sufficient to store
only the URN of the chosen resource rather than all possible aliases. In
addition to preserving the ability to fix up references to aliased
resources, retaining the chosen alias allows the history of a logical
resource to be followed across aliases.
2022-03-28 15:36:08 +00:00
|
|
|
func (j *Journal) Snap(base *deploy.Snapshot) (*deploy.Snapshot, error) {
|
2020-10-15 17:35:09 +00:00
|
|
|
return j.entries.Snap(base)
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewJournal() *Journal {
|
|
|
|
j := &Journal{
|
|
|
|
events: make(chan JournalEntry),
|
|
|
|
cancel: make(chan bool),
|
|
|
|
done: make(chan bool),
|
|
|
|
}
|
|
|
|
go func() {
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-j.cancel:
|
|
|
|
close(j.done)
|
|
|
|
return
|
|
|
|
case e := <-j.events:
|
|
|
|
j.entries = append(j.entries, e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
return j
|
|
|
|
}
|