pulumi/pkg/resource/stack/deployment.go

729 lines
24 KiB
Go
Raw Permalink Normal View History

// Copyright 2016-2022, Pulumi Corporation.
2018-05-22 19:43:36 +00:00
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package stack
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"reflect"
"strings"
"github.com/pulumi/pulumi/pkg/v3/resource/deploy"
"github.com/pulumi/pulumi/pkg/v3/secrets"
"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
"github.com/pulumi/pulumi/sdk/v3/go/common/apitype/migrate"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
Move assets and archives to their own package (#15157) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description This PR is motivated by https://github.com/pulumi/pulumi/pull/15145. `resource.*` should be built on top of `property.Value`,[^1] which means that `resource` needs to be able to import `property.Value`, and so `property` cannot import `resource`. Since Assets and Archives are both types of properties, they must be moved out of `resource`. [^1]: For example: https://github.com/pulumi/pulumi/blob/a1d686227cd7e3c70c51bd772450cb0cd57c1479/sdk/go/common/resource/resource_state.go#L35-L36 ## Open Question This PR moves them to their own sub-folders in `resource`. Should `asset` and `archive` live somewhere more high level, like `sdk/go/property/{asset,archive}`? <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> ## Checklist - [ ] I have run `make tidy` to update any new dependencies - [ ] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-01-25 20:39:31 +00:00
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/archive"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/asset"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
"github.com/pulumi/pulumi/sdk/v3/go/common/slice"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
"github.com/santhosh-tekuri/jsonschema/v5"
)
const (
// DeploymentSchemaVersionOldestSupported is the oldest deployment schema that we
// still support, i.e. we can produce a `deploy.Snapshot` from. This will generally
// need to be at least one less than the current schema version so that old deployments can
// be migrated to the current schema.
DeploymentSchemaVersionOldestSupported = 1
2019-11-21 22:58:30 +00:00
// computedValue is a magic number we emit for a value of a resource.Property value
// whenever we need to serialize a resource.Computed. (Since the real/actual value
// is not known.) This allows us to persist engine events and resource states that
// indicate a value will changed... but is unknown what it will change to.
computedValuePlaceholder = "04da6b54-80e4-46f7-96ec-b56ff0331ba9"
)
var (
// ErrDeploymentSchemaVersionTooOld is returned from `DeserializeDeployment` if the
// untyped deployment being deserialized is too old to understand.
Enable perfsprint linter (#14813) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Prompted by a comment in another review: https://github.com/pulumi/pulumi/pull/14654#discussion_r1419995945 This lints that we don't use `fmt.Errorf` when `errors.New` will suffice, it also covers a load of other cases where `Sprintf` is sub-optimal. Most of these edits were made by running `perfsprint --fix`. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-12 12:19:42 +00:00
ErrDeploymentSchemaVersionTooOld = errors.New("this stack's deployment is too old")
// ErrDeploymentSchemaVersionTooNew is returned from `DeserializeDeployment` if the
// untyped deployment being deserialized is too new to understand.
Enable perfsprint linter (#14813) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Prompted by a comment in another review: https://github.com/pulumi/pulumi/pull/14654#discussion_r1419995945 This lints that we don't use `fmt.Errorf` when `errors.New` will suffice, it also covers a load of other cases where `Sprintf` is sub-optimal. Most of these edits were made by running `perfsprint --fix`. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-12 12:19:42 +00:00
ErrDeploymentSchemaVersionTooNew = errors.New("this stack's deployment version is too new")
)
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
var (
deploymentSchema *jsonschema.Schema
propertyValueSchema *jsonschema.Schema
)
func init() {
compiler := jsonschema.NewCompiler()
compiler.LoadURL = func(s string) (io.ReadCloser, error) {
var schema string
switch s {
case apitype.DeploymentSchemaID:
schema = apitype.DeploymentSchema()
case apitype.ResourceSchemaID:
schema = apitype.ResourceSchema()
case apitype.PropertyValueSchemaID:
schema = apitype.PropertyValueSchema()
default:
return jsonschema.LoadURL(s)
}
return io.NopCloser(strings.NewReader(schema)), nil
}
deploymentSchema = compiler.MustCompile(apitype.DeploymentSchemaID)
propertyValueSchema = compiler.MustCompile(apitype.PropertyValueSchemaID)
}
// ValidateUntypedDeployment validates a deployment against the Deployment JSON schema.
func ValidateUntypedDeployment(deployment *apitype.UntypedDeployment) error {
bytes, err := json.Marshal(deployment)
if err != nil {
return err
}
var raw interface{}
if err := json.Unmarshal(bytes, &raw); err != nil {
return err
}
return deploymentSchema.Validate(raw)
}
// SerializeDeployment serializes an entire snapshot as a deploy record.
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
func SerializeDeployment(ctx context.Context, snap *deploy.Snapshot, showSecrets bool) (*apitype.DeploymentV3, error) {
contract.Requiref(snap != nil, "snap", "must not be nil")
// Capture the version information into a manifest.
manifest := snap.Manifest.Serialize()
Always use the snapshot secret manager (#15768) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> There were a number of places where we passed a `Snapshot` and a `secret.Manager` as arguments to a method, where if the `Manger` was nil we'd fall back to the `Snapshot.SecretManager` (which could also be nil). Turns out in all but one place this was always passed as nil or just as directly the snapshot's `SecretManager` field. The one place it differed was in `pkg/cmd/pulumi/stack_change_secrets_provider.go` where we're changing the secret manager, but it's fine to just set the snapshot's `SecretManager` field to the new manager. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-03-25 10:30:14 +00:00
sm := snap.SecretsManager
var enc config.Encrypter
if sm != nil {
e, err := sm.Encrypter()
if err != nil {
return nil, fmt.Errorf("getting encrypter for deployment: %w", err)
}
enc = e
} else {
enc = config.NewPanicCrypter()
}
// Serialize all vertices and only include a vertex section if non-empty.
resources := slice.Prealloc[apitype.ResourceV3](len(snap.Resources))
for _, res := range snap.Resources {
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
sres, err := SerializeResource(ctx, res, enc, showSecrets)
if err != nil {
return nil, fmt.Errorf("serializing resources: %w", err)
}
resources = append(resources, sres)
}
operations := slice.Prealloc[apitype.OperationV2](len(snap.PendingOperations))
Add a list of in-flight operations to the deployment (#1759) * Add a list of in-flight operations to the deployment This commit augments 'DeploymentV2' with a list of operations that are currently in flight. This information is used by the engine to keep track of whether or not a particular deployment is in a valid state. The SnapshotManager is responsible for inserting and removing operations from the in-flight operation list. When the engine registers an intent to perform an operation, SnapshotManager inserts an Operation into this list and saves it to the snapshot. When an operation completes, the SnapshotManager removes it from the snapshot. From this, the engine can infer that if it ever sees a deployment with pending operations, the Pulumi CLI must have crashed or otherwise abnormally terminated before seeing whether or not an operation completed successfully. To remedy this state, this commit also adds code to 'pulumi stack import' that clears all pending operations from a deployment, as well as code to plan generation that will reject any deployments that have pending operations present. At the CLI level, if we see that we are in a state where pending operations were in-flight when the engine died, we'll issue a human-friendly error message that indicates which resources are in a bad state and how to recover their stack. * CR: Multi-line string literals, renaming in-flight -> pending * CR: Add enum to apitype for operation type, also name status -> type for clarity * Fix the yaml type * Fix missed renames * Add implementation for lifecycle_test.go * Rebase against master
2018-08-11 04:39:59 +00:00
for _, op := range snap.PendingOperations {
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
sop, err := SerializeOperation(ctx, op, enc, showSecrets)
if err != nil {
return nil, err
}
operations = append(operations, sop)
}
var secretsProvider *apitype.SecretsProvidersV1
if sm != nil {
secretsProvider = &apitype.SecretsProvidersV1{
Type: sm.Type(),
State: sm.State(),
}
Add a list of in-flight operations to the deployment (#1759) * Add a list of in-flight operations to the deployment This commit augments 'DeploymentV2' with a list of operations that are currently in flight. This information is used by the engine to keep track of whether or not a particular deployment is in a valid state. The SnapshotManager is responsible for inserting and removing operations from the in-flight operation list. When the engine registers an intent to perform an operation, SnapshotManager inserts an Operation into this list and saves it to the snapshot. When an operation completes, the SnapshotManager removes it from the snapshot. From this, the engine can infer that if it ever sees a deployment with pending operations, the Pulumi CLI must have crashed or otherwise abnormally terminated before seeing whether or not an operation completed successfully. To remedy this state, this commit also adds code to 'pulumi stack import' that clears all pending operations from a deployment, as well as code to plan generation that will reject any deployments that have pending operations present. At the CLI level, if we see that we are in a state where pending operations were in-flight when the engine died, we'll issue a human-friendly error message that indicates which resources are in a bad state and how to recover their stack. * CR: Multi-line string literals, renaming in-flight -> pending * CR: Add enum to apitype for operation type, also name status -> type for clarity * Fix the yaml type * Fix missed renames * Add implementation for lifecycle_test.go * Rebase against master
2018-08-11 04:39:59 +00:00
}
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
return &apitype.DeploymentV3{
Add a list of in-flight operations to the deployment (#1759) * Add a list of in-flight operations to the deployment This commit augments 'DeploymentV2' with a list of operations that are currently in flight. This information is used by the engine to keep track of whether or not a particular deployment is in a valid state. The SnapshotManager is responsible for inserting and removing operations from the in-flight operation list. When the engine registers an intent to perform an operation, SnapshotManager inserts an Operation into this list and saves it to the snapshot. When an operation completes, the SnapshotManager removes it from the snapshot. From this, the engine can infer that if it ever sees a deployment with pending operations, the Pulumi CLI must have crashed or otherwise abnormally terminated before seeing whether or not an operation completed successfully. To remedy this state, this commit also adds code to 'pulumi stack import' that clears all pending operations from a deployment, as well as code to plan generation that will reject any deployments that have pending operations present. At the CLI level, if we see that we are in a state where pending operations were in-flight when the engine died, we'll issue a human-friendly error message that indicates which resources are in a bad state and how to recover their stack. * CR: Multi-line string literals, renaming in-flight -> pending * CR: Add enum to apitype for operation type, also name status -> type for clarity * Fix the yaml type * Fix missed renames * Add implementation for lifecycle_test.go * Rebase against master
2018-08-11 04:39:59 +00:00
Manifest: manifest,
Resources: resources,
SecretsProviders: secretsProvider,
Add a list of in-flight operations to the deployment (#1759) * Add a list of in-flight operations to the deployment This commit augments 'DeploymentV2' with a list of operations that are currently in flight. This information is used by the engine to keep track of whether or not a particular deployment is in a valid state. The SnapshotManager is responsible for inserting and removing operations from the in-flight operation list. When the engine registers an intent to perform an operation, SnapshotManager inserts an Operation into this list and saves it to the snapshot. When an operation completes, the SnapshotManager removes it from the snapshot. From this, the engine can infer that if it ever sees a deployment with pending operations, the Pulumi CLI must have crashed or otherwise abnormally terminated before seeing whether or not an operation completed successfully. To remedy this state, this commit also adds code to 'pulumi stack import' that clears all pending operations from a deployment, as well as code to plan generation that will reject any deployments that have pending operations present. At the CLI level, if we see that we are in a state where pending operations were in-flight when the engine died, we'll issue a human-friendly error message that indicates which resources are in a bad state and how to recover their stack. * CR: Multi-line string literals, renaming in-flight -> pending * CR: Add enum to apitype for operation type, also name status -> type for clarity * Fix the yaml type * Fix missed renames * Add implementation for lifecycle_test.go * Rebase against master
2018-08-11 04:39:59 +00:00
PendingOperations: operations,
}, nil
}
Restore secrets provider in config refresh (#13900) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/7282 This is fairly simple, just grab the last deployment from the stack (we should have one otherwise we wouldn't have any config to fetch either) and pull the SecretsProviders data out the deployment data and translate and insert it into the stack config. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - I've manually checked this with a passphrase deployment. Need to do the command split for "config refresh" to write up some unit tests to cover this. <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-11-13 12:27:46 +00:00
// UnmarshalUntypedDeployment unmarshals a raw untyped deployment into an up to date deployment object.
func UnmarshalUntypedDeployment(
ctx context.Context,
deployment *apitype.UntypedDeployment,
Restore secrets provider in config refresh (#13900) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/7282 This is fairly simple, just grab the last deployment from the stack (we should have one otherwise we wouldn't have any config to fetch either) and pull the SecretsProviders data out the deployment data and translate and insert it into the stack config. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - I've manually checked this with a passphrase deployment. Need to do the command split for "config refresh" to write up some unit tests to cover this. <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-11-13 12:27:46 +00:00
) (*apitype.DeploymentV3, error) {
contract.Requiref(deployment != nil, "deployment", "must not be nil")
switch {
case deployment.Version > apitype.DeploymentSchemaVersionCurrent:
return nil, ErrDeploymentSchemaVersionTooNew
case deployment.Version < DeploymentSchemaVersionOldestSupported:
return nil, ErrDeploymentSchemaVersionTooOld
}
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
var v3deployment apitype.DeploymentV3
switch deployment.Version {
case 1:
var v1deployment apitype.DeploymentV1
if err := json.Unmarshal([]byte(deployment.Deployment), &v1deployment); err != nil {
return nil, err
}
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
v2deployment := migrate.UpToDeploymentV2(v1deployment)
v3deployment = migrate.UpToDeploymentV3(v2deployment)
case 2:
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
var v2deployment apitype.DeploymentV2
if err := json.Unmarshal([]byte(deployment.Deployment), &v2deployment); err != nil {
return nil, err
}
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
v3deployment = migrate.UpToDeploymentV3(v2deployment)
case 3:
if err := json.Unmarshal([]byte(deployment.Deployment), &v3deployment); err != nil {
return nil, err
}
default:
contract.Failf("unrecognized version: %d", deployment.Version)
}
Restore secrets provider in config refresh (#13900) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/7282 This is fairly simple, just grab the last deployment from the stack (we should have one otherwise we wouldn't have any config to fetch either) and pull the SecretsProviders data out the deployment data and translate and insert it into the stack config. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - I've manually checked this with a passphrase deployment. Need to do the command split for "config refresh" to write up some unit tests to cover this. <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-11-13 12:27:46 +00:00
return &v3deployment, nil
}
// DeserializeUntypedDeployment deserializes an untyped deployment and produces a `deploy.Snapshot`
// from it. DeserializeDeployment will return an error if the untyped deployment's version is
// not within the range `DeploymentSchemaVersionCurrent` and `DeploymentSchemaVersionOldestSupported`.
func DeserializeUntypedDeployment(
ctx context.Context,
deployment *apitype.UntypedDeployment,
secretsProv secrets.Provider,
) (*deploy.Snapshot, error) {
v3deployment, err := UnmarshalUntypedDeployment(ctx, deployment)
if err != nil {
return nil, err
}
return DeserializeDeploymentV3(ctx, *v3deployment, secretsProv)
}
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
// DeserializeDeploymentV3 deserializes a typed DeploymentV3 into a `deploy.Snapshot`.
func DeserializeDeploymentV3(
ctx context.Context,
deployment apitype.DeploymentV3,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
secretsProv secrets.Provider,
) (*deploy.Snapshot, error) {
// Unpack the versions.
manifest, err := deploy.DeserializeManifest(deployment.Manifest)
if err != nil {
return nil, err
}
var secretsManager secrets.Manager
if deployment.SecretsProviders != nil && deployment.SecretsProviders.Type != "" {
if secretsProv == nil {
return nil, errors.New("deployment uses a SecretsProvider but no SecretsProvider was provided")
}
sm, err := secretsProv.OfType(deployment.SecretsProviders.Type, deployment.SecretsProviders.State)
if err != nil {
return nil, err
}
secretsManager = sm
}
var dec config.Decrypter
var enc config.Encrypter
if secretsManager == nil {
var ciphertexts []string
for _, res := range deployment.Resources {
collectCiphertexts(&ciphertexts, res.Inputs)
collectCiphertexts(&ciphertexts, res.Outputs)
}
if len(ciphertexts) > 0 {
// If there are ciphertexts, but we couldn't set up a secrets manager, error out early
// to avoid panic'ing later on. This snapshot is broken and needs to be repaired
// manually.
return nil, errors.New("snapshot contains encrypted secrets but no secrets manager could be found")
}
dec = config.NewPanicCrypter()
enc = config.NewPanicCrypter()
} else {
d, err := secretsManager.Decrypter()
if err != nil {
return nil, err
}
// Do a first pass through state and collect all of the secrets that need decrypting.
// We will collect all secrets and decrypt them all at once, rather than just-in-time.
// We do this to avoid serial calls to the decryption endpoint which can result in long
// wait times in stacks with a large number of secrets.
var ciphertexts []string
for _, res := range deployment.Resources {
collectCiphertexts(&ciphertexts, res.Inputs)
collectCiphertexts(&ciphertexts, res.Outputs)
}
// Decrypt the collected secrets and create a decrypter that will use the result as a cache.
cache, err := d.BulkDecrypt(ctx, ciphertexts)
if err != nil {
return nil, err
}
dec = newMapDecrypter(d, cache)
e, err := secretsManager.Encrypter()
if err != nil {
return nil, err
}
enc = e
}
// For every serialized resource vertex, create a ResourceDeployment out of it.
resources := slice.Prealloc[*resource.State](len(deployment.Resources))
for _, res := range deployment.Resources {
desres, err := DeserializeResource(res, dec, enc)
if err != nil {
return nil, err
}
resources = append(resources, desres)
}
ops := slice.Prealloc[resource.Operation](len(deployment.PendingOperations))
Add a list of in-flight operations to the deployment (#1759) * Add a list of in-flight operations to the deployment This commit augments 'DeploymentV2' with a list of operations that are currently in flight. This information is used by the engine to keep track of whether or not a particular deployment is in a valid state. The SnapshotManager is responsible for inserting and removing operations from the in-flight operation list. When the engine registers an intent to perform an operation, SnapshotManager inserts an Operation into this list and saves it to the snapshot. When an operation completes, the SnapshotManager removes it from the snapshot. From this, the engine can infer that if it ever sees a deployment with pending operations, the Pulumi CLI must have crashed or otherwise abnormally terminated before seeing whether or not an operation completed successfully. To remedy this state, this commit also adds code to 'pulumi stack import' that clears all pending operations from a deployment, as well as code to plan generation that will reject any deployments that have pending operations present. At the CLI level, if we see that we are in a state where pending operations were in-flight when the engine died, we'll issue a human-friendly error message that indicates which resources are in a bad state and how to recover their stack. * CR: Multi-line string literals, renaming in-flight -> pending * CR: Add enum to apitype for operation type, also name status -> type for clarity * Fix the yaml type * Fix missed renames * Add implementation for lifecycle_test.go * Rebase against master
2018-08-11 04:39:59 +00:00
for _, op := range deployment.PendingOperations {
desop, err := DeserializeOperation(op, dec, enc)
Add a list of in-flight operations to the deployment (#1759) * Add a list of in-flight operations to the deployment This commit augments 'DeploymentV2' with a list of operations that are currently in flight. This information is used by the engine to keep track of whether or not a particular deployment is in a valid state. The SnapshotManager is responsible for inserting and removing operations from the in-flight operation list. When the engine registers an intent to perform an operation, SnapshotManager inserts an Operation into this list and saves it to the snapshot. When an operation completes, the SnapshotManager removes it from the snapshot. From this, the engine can infer that if it ever sees a deployment with pending operations, the Pulumi CLI must have crashed or otherwise abnormally terminated before seeing whether or not an operation completed successfully. To remedy this state, this commit also adds code to 'pulumi stack import' that clears all pending operations from a deployment, as well as code to plan generation that will reject any deployments that have pending operations present. At the CLI level, if we see that we are in a state where pending operations were in-flight when the engine died, we'll issue a human-friendly error message that indicates which resources are in a bad state and how to recover their stack. * CR: Multi-line string literals, renaming in-flight -> pending * CR: Add enum to apitype for operation type, also name status -> type for clarity * Fix the yaml type * Fix missed renames * Add implementation for lifecycle_test.go * Rebase against master
2018-08-11 04:39:59 +00:00
if err != nil {
return nil, err
}
ops = append(ops, desop)
}
return deploy.NewSnapshot(*manifest, secretsManager, resources, ops), nil
}
// SerializeResource turns a resource into a structure suitable for serialization.
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
func SerializeResource(
ctx context.Context, res *resource.State, enc config.Encrypter, showSecrets bool,
) (apitype.ResourceV3, error) {
contract.Requiref(res != nil, "res", "must not be nil")
contract.Requiref(res.URN != "", "res", "must have a URN")
Revert "Revert "Run integration tests and dev builds with race detection" (#15998)" (#16148) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This reverts commit 75340dd94203da02e44ca5f8beb55d9063d302ef. Fixes https://github.com/pulumi/pulumi/issues/16018. This re-enables the locking and race detection. The locking is more finely scoped to not be held over provider methods like Read/Update. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-05-09 16:15:41 +00:00
res.Lock.Lock()
defer res.Lock.Unlock()
// Serialize all input and output properties recursively, and add them if non-empty.
var inputs map[string]interface{}
if inp := res.Inputs; inp != nil {
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
sinp, err := SerializeProperties(ctx, inp, enc, showSecrets)
if err != nil {
return apitype.ResourceV3{}, err
}
inputs = sinp
}
var outputs map[string]interface{}
if outp := res.Outputs; outp != nil {
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
soutp, err := SerializeProperties(ctx, outp, enc, showSecrets)
if err != nil {
return apitype.ResourceV3{}, err
}
outputs = soutp
}
v3Resource := apitype.ResourceV3{
URN: res.URN,
Custom: res.Custom,
Delete: res.Delete,
ID: res.ID,
Type: res.Type,
Parent: res.Parent,
Inputs: inputs,
Outputs: outputs,
Protect: res.Protect,
External: res.External,
Dependencies: res.Dependencies,
InitErrors: res.InitErrors,
Provider: res.Provider,
PropertyDependencies: res.PropertyDependencies,
PendingReplacement: res.PendingReplacement,
AdditionalSecretOutputs: res.AdditionalSecretOutputs,
Aliases: res.Aliases,
ImportID: res.ImportID,
RetainOnDelete: res.RetainOnDelete,
DeletedWith: res.DeletedWith,
Created: res.Created,
Modified: res.Modified,
[engine] Add support for source positions These changes add support for passing source position information in gRPC metadata and recording the source position that corresponds to a resource registration in the statefile. Enabling source position information in the resource model can provide substantial benefits, including but not limited to: - Better errors from the Pulumi CLI - Go-to-defintion for resources in state - Editor integration for errors, etc. from `pulumi preview` Source positions are (file, line) or (file, line, column) tuples represented as URIs. The line and column are stored in the fragment portion of the URI as "line(,column)?". The scheme of the URI and the form of its path component depends on the context in which it is generated or used: - During an active update, the URI's scheme is `file` and paths are absolute filesystem paths. This allows consumers to easily access arbitrary files that are available on the host. - In a statefile, the URI's scheme is `project` and paths are relative to the project root. This allows consumers to resolve source positions relative to the project file in different contexts irrespective of the location of the project itself (e.g. given a project-relative path and the URL of the project's root on GitHub, one can build a GitHub URL for the source position). During an update, source position information may be attached to gRPC calls as "source-position" metadata. This allows arbitrary calls to be associated with source positions without changes to their protobuf payloads. Modifying the protobuf payloads is also a viable approach, but is somewhat more invasive than attaching metadata, and requires changes to every call signature. Source positions should reflect the position in user code that initiated a resource model operation (e.g. the source position passed with `RegisterResource` for `pet` in the example above should be the source position in `index.ts`, _not_ the source position in the Pulumi SDK). In general, the Pulumi SDK should be able to infer the source position of the resource registration, as the relationship between a resource registration and its corresponding user code should be static per SDK. Source positions in state files will be stored as a new `registeredAt` property on each resource. This property is optional.
2023-06-29 18:41:19 +00:00
SourcePosition: res.SourcePosition,
Change `pulumi refresh` to report diff relative to desired state instead of relative to only output changes (#16146) Presently, the behaviour of diffing during refresh steps is incomplete, returning only an "output diff" that presents the changes in outputs. This commit changes refresh steps so that: * they compute a diff similar to the one that would be computed if a `preview` were run immediately after the refresh, which is more typically what users expect and want; and * `IgnoreChanges` resource options are respected when performing the new desired-state diffs, so that property additions or changes reported by a refresh can be ignored. In particular, `IgnoreChanges` can now be used to acknowledge that part or all of a resource may change in the provider, but the user is OK with this and doesn't want to be notified about it during a refresh. Importantly, this means that the diff won't be reported, but also that the changes won't be applied to state. The implementation covers the following: * A diff is computed using the inputs from the program and then inverting the result, since in the case of a refresh the diff is being driven by the provider side and not the program. This doesn't change what is stored back into the state, but it does produce a diff that is more aligned with the "true changes to the desired state". * `IgnoreChanges` resource options are now stored in state, so that this information can be used in refresh operations that do not have access to/run the program. * In the context of a refresh operation, `IgnoreChanges` applies to *both* input and output properties. This differs from the behaviour of a normal update operation, where `IgnoreChanges` only considers input properties. * The special `"*"` value for `IgnoreChanges` can be used to ignore all properties. It _also_ ignores the case where the resource cannot be found in the provider, and instead keeps the resource intact in state with its existing input and output properties. Because the program is not run for refresh operations, `IgnoreChanges` options must be applied separately before a refresh takes place. This can be accomplished using e.g. a `pulumi up` that applies the options prior to a refresh. We should investigate perhaps providing a `pulumi state set ...`-like CLI to make these sorts of changes directly to a state. For use cases relying on the legacy refresh diff provider, the `PULUMI_USE_LEGACY_REFRESH_DIFF` environment variable can be set, which will disable desired-state diff computation. We only need to perform checks in `RefreshStep.{ResultOp,Apply}`, since downstream code will work correctly based on the presence or absence of a `DetailedDiff` in the step. ### Notes - https://github.com/pulumi/pulumi/issues/16144 affects some of these cases - though its technically orthogonal - https://github.com/pulumi/pulumi/issues/11279 is another technically orthogonal issue that many providers (at least TFBridge ones) - do not report back changes to input properties on Read when the input property (or property path) was missing on the inputs. This is again technically orthogonal - but leads to cases that appear "wrong" in terms of what is stored back into the state still - though the same as before this change. - Azure Native doesn't seem to handle `ignoreChanges` passed to Diff, so the ability to ignore changes on refresh doesn't currently work for Azure Native. ### Fixes * Fixes #16072 * Fixes #16278 * Fixes #16334 * Not quite #12346, but likely replaces the need for that Co-authored-by: Will Jones <will@sacharissa.co.uk>
2024-06-12 16:17:05 +00:00
IgnoreChanges: res.IgnoreChanges,
}
if res.CustomTimeouts.IsNotEmpty() {
v3Resource.CustomTimeouts = &res.CustomTimeouts
}
return v3Resource, nil
}
// SerializeOperation serializes a resource in a pending state.
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
func SerializeOperation(
ctx context.Context, op resource.Operation, enc config.Encrypter, showSecrets bool,
) (apitype.OperationV2, error) {
res, err := SerializeResource(ctx, op.Resource, enc, showSecrets)
if err != nil {
return apitype.OperationV2{}, fmt.Errorf("serializing resource: %w", err)
}
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
return apitype.OperationV2{
Add a list of in-flight operations to the deployment (#1759) * Add a list of in-flight operations to the deployment This commit augments 'DeploymentV2' with a list of operations that are currently in flight. This information is used by the engine to keep track of whether or not a particular deployment is in a valid state. The SnapshotManager is responsible for inserting and removing operations from the in-flight operation list. When the engine registers an intent to perform an operation, SnapshotManager inserts an Operation into this list and saves it to the snapshot. When an operation completes, the SnapshotManager removes it from the snapshot. From this, the engine can infer that if it ever sees a deployment with pending operations, the Pulumi CLI must have crashed or otherwise abnormally terminated before seeing whether or not an operation completed successfully. To remedy this state, this commit also adds code to 'pulumi stack import' that clears all pending operations from a deployment, as well as code to plan generation that will reject any deployments that have pending operations present. At the CLI level, if we see that we are in a state where pending operations were in-flight when the engine died, we'll issue a human-friendly error message that indicates which resources are in a bad state and how to recover their stack. * CR: Multi-line string literals, renaming in-flight -> pending * CR: Add enum to apitype for operation type, also name status -> type for clarity * Fix the yaml type * Fix missed renames * Add implementation for lifecycle_test.go * Rebase against master
2018-08-11 04:39:59 +00:00
Resource: res,
Type: apitype.OperationType(op.Type),
}, nil
Add a list of in-flight operations to the deployment (#1759) * Add a list of in-flight operations to the deployment This commit augments 'DeploymentV2' with a list of operations that are currently in flight. This information is used by the engine to keep track of whether or not a particular deployment is in a valid state. The SnapshotManager is responsible for inserting and removing operations from the in-flight operation list. When the engine registers an intent to perform an operation, SnapshotManager inserts an Operation into this list and saves it to the snapshot. When an operation completes, the SnapshotManager removes it from the snapshot. From this, the engine can infer that if it ever sees a deployment with pending operations, the Pulumi CLI must have crashed or otherwise abnormally terminated before seeing whether or not an operation completed successfully. To remedy this state, this commit also adds code to 'pulumi stack import' that clears all pending operations from a deployment, as well as code to plan generation that will reject any deployments that have pending operations present. At the CLI level, if we see that we are in a state where pending operations were in-flight when the engine died, we'll issue a human-friendly error message that indicates which resources are in a bad state and how to recover their stack. * CR: Multi-line string literals, renaming in-flight -> pending * CR: Add enum to apitype for operation type, also name status -> type for clarity * Fix the yaml type * Fix missed renames * Add implementation for lifecycle_test.go * Rebase against master
2018-08-11 04:39:59 +00:00
}
// SerializeProperties serializes a resource property bag so that it's suitable for serialization.
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
func SerializeProperties(ctx context.Context, props resource.PropertyMap, enc config.Encrypter,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
showSecrets bool,
) (map[string]interface{}, error) {
dst := make(map[string]interface{})
for _, k := range props.StableKeys() {
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
v, err := SerializePropertyValue(ctx, props[k], enc, showSecrets)
if err != nil {
return nil, err
}
dst[string(k)] = v
}
return dst, nil
}
// SerializePropertyValue serializes a resource property value so that it's suitable for serialization.
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
func SerializePropertyValue(ctx context.Context, prop resource.PropertyValue, enc config.Encrypter,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
showSecrets bool,
) (interface{}, error) {
// Serialize nulls as nil.
if prop.IsNull() {
return nil, nil
}
2019-11-21 22:58:30 +00:00
// A computed value marks something that will be determined at a later time. (e.g. the result of
// a computation that we don't perform during a preview operation.) We serialize a magic constant
// to record its existence.
if prop.IsComputed() || prop.IsOutput() {
2019-11-21 22:58:30 +00:00
return computedValuePlaceholder, nil
}
// For arrays, make sure to recurse.
if prop.IsArray() {
srcarr := prop.ArrayValue()
dstarr := make([]interface{}, len(srcarr))
for i, elem := range prop.ArrayValue() {
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
selem, err := SerializePropertyValue(ctx, elem, enc, showSecrets)
if err != nil {
return nil, err
}
dstarr[i] = selem
}
return dstarr, nil
}
// Also for objects, recurse and use naked properties.
if prop.IsObject() {
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
return SerializeProperties(ctx, prop.ObjectValue(), enc, showSecrets)
}
// For assets, we need to serialize them a little carefully, so we can recover them afterwards.
if prop.IsAsset() {
return prop.AssetValue().Serialize(), nil
} else if prop.IsArchive() {
return prop.ArchiveValue().Serialize(), nil
}
// We serialize resource references using a map-based representation similar to assets, archives, and secrets.
if prop.IsResourceReference() {
ref := prop.ResourceReferenceValue()
serialized := map[string]interface{}{
resource.SigKey: resource.ResourceReferenceSig,
"urn": string(ref.URN),
"packageVersion": ref.PackageVersion,
}
if id, hasID := ref.IDString(); hasID {
serialized["id"] = id
}
return serialized, nil
}
if prop.IsSecret() {
// Since we are going to encrypt property value, we can elide encrypting sub-elements. We'll mark them as
// "secret" so we retain that information when deserializing the overall structure, but there is no
// need to double encrypt everything.
Lift context parameter to SerializeDeployment/Resource/Operations/Properties (#15929) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> SerializePropertyValue needed a `context.Context` object to pass to the `config.Encrypter`. It was using `context.TODO()`, this change instead accepts a context on the parameters and lifts that up to SerializeProperties, SerializeResource, SerializeOperation, and SerializeDeployment. There were a few call sites for those methods that already had a context on hand, and they now pass that context. The other calls sites now use `context.TODO()`, we should continue to iterate in this area to ensure everywhere that needs a context has one passed in. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-15 07:45:46 +00:00
value, err := SerializePropertyValue(ctx, prop.SecretValue().Element, config.NopEncrypter, showSecrets)
if err != nil {
return nil, err
}
bytes, err := json.Marshal(value)
if err != nil {
return nil, fmt.Errorf("encoding serialized property value: %w", err)
}
plaintext := string(bytes)
secret := apitype.SecretV1{
Sig: resource.SecretSig,
}
if showSecrets {
secret.Plaintext = plaintext
} else {
Don't encrypt secrets to get ciphertext that isn't actually used (#15306) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> While investigating into https://github.com/pulumi/pulumi/issues/15293 I noticed that `SerializePropertyValue` was always calling into `Encrypter.EncryptValue` (or `cachingCrypter.encryptSecret`) even when `showSecrets` was true and the cipher text wasn't being returned. This is a simple move of that logic into the `!showSecrets` branch so it gets skipped entirely when the cipher text isn't needed. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-01-30 09:51:51 +00:00
// If the encrypter is a cachingCrypter, call through its encryptSecret method, which will look for a matching
// *resource.Secret + plaintext in its cache in order to avoid re-encrypting the value.
var ciphertext string
if cachingCrypter, ok := enc.(*cachingCrypter); ok {
Add `context.Context` to `cachingCrypter.encryptSecret` (#15765) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Was looking at secret code and saw this was an easy fix up of a `context.TODO()` call. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-03-25 10:27:47 +00:00
ciphertext, err = cachingCrypter.encryptSecret(ctx, prop.SecretValue(), plaintext)
Don't encrypt secrets to get ciphertext that isn't actually used (#15306) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> While investigating into https://github.com/pulumi/pulumi/issues/15293 I noticed that `SerializePropertyValue` was always calling into `Encrypter.EncryptValue` (or `cachingCrypter.encryptSecret`) even when `showSecrets` was true and the cipher text wasn't being returned. This is a simple move of that logic into the `!showSecrets` branch so it gets skipped entirely when the cipher text isn't needed. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-01-30 09:51:51 +00:00
} else {
ciphertext, err = enc.EncryptValue(ctx, plaintext)
}
if err != nil {
return nil, fmt.Errorf("failed to encrypt secret value: %w", err)
}
contract.AssertNoErrorf(err, "marshalling underlying secret value to JSON")
secret.Ciphertext = ciphertext
}
return secret, nil
}
// All others are returned as-is.
return prop.V, nil
}
// collectCiphertexts collects encrypted secrets from resource properties.
func collectCiphertexts(ciphertexts *[]string, prop interface{}) {
switch prop := prop.(type) {
case []interface{}:
for _, v := range prop {
collectCiphertexts(ciphertexts, v)
}
case map[string]interface{}:
if prop[resource.SigKey] == resource.SecretSig {
if ciphertext, cipherOk := prop["ciphertext"].(string); cipherOk {
*ciphertexts = append(*ciphertexts, ciphertext)
}
} else {
for _, v := range prop {
collectCiphertexts(ciphertexts, v)
}
}
}
}
// DeserializeResource turns a serialized resource back into its usual form.
func DeserializeResource(res apitype.ResourceV3, dec config.Decrypter, enc config.Encrypter) (*resource.State, error) {
// Deserialize the resource properties, if they exist.
inputs, err := DeserializeProperties(res.Inputs, dec, enc)
if err != nil {
return nil, err
}
outputs, err := DeserializeProperties(res.Outputs, dec, enc)
if err != nil {
return nil, err
}
if res.URN == "" {
Enable perfsprint linter (#14813) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Prompted by a comment in another review: https://github.com/pulumi/pulumi/pull/14654#discussion_r1419995945 This lints that we don't use `fmt.Errorf` when `errors.New` will suffice, it also covers a load of other cases where `Sprintf` is sub-optimal. Most of these edits were made by running `perfsprint --fix`. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-12 12:19:42 +00:00
return nil, errors.New("resource missing required 'urn' field")
}
if res.Type == "" {
return nil, fmt.Errorf("resource '%s' missing required 'type' field", res.URN)
}
if !res.Custom && res.ID != "" {
return nil, fmt.Errorf("resource '%s' has 'custom' false but non-empty ID", res.URN)
}
return resource.NewState(
res.Type, res.URN, res.Custom, res.Delete, res.ID,
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
inputs, outputs, res.Parent, res.Protect, res.External, res.Dependencies, res.InitErrors, res.Provider,
res.PropertyDependencies, res.PendingReplacement, res.AdditionalSecretOutputs, res.Aliases, res.CustomTimeouts,
Change `pulumi refresh` to report diff relative to desired state instead of relative to only output changes (#16146) Presently, the behaviour of diffing during refresh steps is incomplete, returning only an "output diff" that presents the changes in outputs. This commit changes refresh steps so that: * they compute a diff similar to the one that would be computed if a `preview` were run immediately after the refresh, which is more typically what users expect and want; and * `IgnoreChanges` resource options are respected when performing the new desired-state diffs, so that property additions or changes reported by a refresh can be ignored. In particular, `IgnoreChanges` can now be used to acknowledge that part or all of a resource may change in the provider, but the user is OK with this and doesn't want to be notified about it during a refresh. Importantly, this means that the diff won't be reported, but also that the changes won't be applied to state. The implementation covers the following: * A diff is computed using the inputs from the program and then inverting the result, since in the case of a refresh the diff is being driven by the provider side and not the program. This doesn't change what is stored back into the state, but it does produce a diff that is more aligned with the "true changes to the desired state". * `IgnoreChanges` resource options are now stored in state, so that this information can be used in refresh operations that do not have access to/run the program. * In the context of a refresh operation, `IgnoreChanges` applies to *both* input and output properties. This differs from the behaviour of a normal update operation, where `IgnoreChanges` only considers input properties. * The special `"*"` value for `IgnoreChanges` can be used to ignore all properties. It _also_ ignores the case where the resource cannot be found in the provider, and instead keeps the resource intact in state with its existing input and output properties. Because the program is not run for refresh operations, `IgnoreChanges` options must be applied separately before a refresh takes place. This can be accomplished using e.g. a `pulumi up` that applies the options prior to a refresh. We should investigate perhaps providing a `pulumi state set ...`-like CLI to make these sorts of changes directly to a state. For use cases relying on the legacy refresh diff provider, the `PULUMI_USE_LEGACY_REFRESH_DIFF` environment variable can be set, which will disable desired-state diff computation. We only need to perform checks in `RefreshStep.{ResultOp,Apply}`, since downstream code will work correctly based on the presence or absence of a `DetailedDiff` in the step. ### Notes - https://github.com/pulumi/pulumi/issues/16144 affects some of these cases - though its technically orthogonal - https://github.com/pulumi/pulumi/issues/11279 is another technically orthogonal issue that many providers (at least TFBridge ones) - do not report back changes to input properties on Read when the input property (or property path) was missing on the inputs. This is again technically orthogonal - but leads to cases that appear "wrong" in terms of what is stored back into the state still - though the same as before this change. - Azure Native doesn't seem to handle `ignoreChanges` passed to Diff, so the ability to ignore changes on refresh doesn't currently work for Azure Native. ### Fixes * Fixes #16072 * Fixes #16278 * Fixes #16334 * Not quite #12346, but likely replaces the need for that Co-authored-by: Will Jones <will@sacharissa.co.uk>
2024-06-12 16:17:05 +00:00
res.ImportID, res.RetainOnDelete, res.DeletedWith, res.Created, res.Modified, res.SourcePosition, res.IgnoreChanges,
), nil
}
// DeserializeOperation hydrates a pending resource/operation pair.
func DeserializeOperation(op apitype.OperationV2, dec config.Decrypter,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
enc config.Encrypter,
) (resource.Operation, error) {
res, err := DeserializeResource(op.Resource, dec, enc)
Add a list of in-flight operations to the deployment (#1759) * Add a list of in-flight operations to the deployment This commit augments 'DeploymentV2' with a list of operations that are currently in flight. This information is used by the engine to keep track of whether or not a particular deployment is in a valid state. The SnapshotManager is responsible for inserting and removing operations from the in-flight operation list. When the engine registers an intent to perform an operation, SnapshotManager inserts an Operation into this list and saves it to the snapshot. When an operation completes, the SnapshotManager removes it from the snapshot. From this, the engine can infer that if it ever sees a deployment with pending operations, the Pulumi CLI must have crashed or otherwise abnormally terminated before seeing whether or not an operation completed successfully. To remedy this state, this commit also adds code to 'pulumi stack import' that clears all pending operations from a deployment, as well as code to plan generation that will reject any deployments that have pending operations present. At the CLI level, if we see that we are in a state where pending operations were in-flight when the engine died, we'll issue a human-friendly error message that indicates which resources are in a bad state and how to recover their stack. * CR: Multi-line string literals, renaming in-flight -> pending * CR: Add enum to apitype for operation type, also name status -> type for clarity * Fix the yaml type * Fix missed renames * Add implementation for lifecycle_test.go * Rebase against master
2018-08-11 04:39:59 +00:00
if err != nil {
return resource.Operation{}, err
}
return resource.NewOperation(res, resource.OperationType(op.Type)), nil
}
// DeserializeProperties deserializes an entire map of deploy properties into a resource property map.
func DeserializeProperties(props map[string]interface{}, dec config.Decrypter,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
enc config.Encrypter,
) (resource.PropertyMap, error) {
result := make(resource.PropertyMap)
for k, prop := range props {
desprop, err := DeserializePropertyValue(prop, dec, enc)
if err != nil {
return nil, err
}
result[resource.PropertyKey(k)] = desprop
}
return result, nil
}
// DeserializePropertyValue deserializes a single deploy property into a resource property value.
func DeserializePropertyValue(v interface{}, dec config.Decrypter,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
enc config.Encrypter,
) (resource.PropertyValue, error) {
ctx := context.TODO()
if v != nil {
switch w := v.(type) {
case bool:
return resource.NewBoolProperty(w), nil
case float64:
return resource.NewNumberProperty(w), nil
case string:
2019-11-21 22:58:30 +00:00
if w == computedValuePlaceholder {
return resource.MakeComputed(resource.NewStringProperty("")), nil
}
return resource.NewStringProperty(w), nil
case []interface{}:
arr := make([]resource.PropertyValue, len(w))
for i, elem := range w {
ev, err := DeserializePropertyValue(elem, dec, enc)
if err != nil {
return resource.PropertyValue{}, err
}
arr[i] = ev
}
return resource.NewArrayProperty(arr), nil
case map[string]interface{}:
obj, err := DeserializeProperties(w, dec, enc)
if err != nil {
return resource.PropertyValue{}, err
}
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
// This could be an asset or archive; if so, recover its type.
objmap := obj.Mappable()
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
if sig, hasSig := objmap[resource.SigKey]; hasSig {
switch sig {
Move assets and archives to their own package (#15157) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description This PR is motivated by https://github.com/pulumi/pulumi/pull/15145. `resource.*` should be built on top of `property.Value`,[^1] which means that `resource` needs to be able to import `property.Value`, and so `property` cannot import `resource`. Since Assets and Archives are both types of properties, they must be moved out of `resource`. [^1]: For example: https://github.com/pulumi/pulumi/blob/a1d686227cd7e3c70c51bd772450cb0cd57c1479/sdk/go/common/resource/resource_state.go#L35-L36 ## Open Question This PR moves them to their own sub-folders in `resource`. Should `asset` and `archive` live somewhere more high level, like `sdk/go/property/{asset,archive}`? <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> ## Checklist - [ ] I have run `make tidy` to update any new dependencies - [ ] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-01-25 20:39:31 +00:00
case asset.AssetSig:
asset, isasset, err := asset.Deserialize(objmap)
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
if err != nil {
return resource.PropertyValue{}, err
}
contract.Assertf(isasset, "resource with asset signature is not an asset")
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
return resource.NewAssetProperty(asset), nil
Move assets and archives to their own package (#15157) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description This PR is motivated by https://github.com/pulumi/pulumi/pull/15145. `resource.*` should be built on top of `property.Value`,[^1] which means that `resource` needs to be able to import `property.Value`, and so `property` cannot import `resource`. Since Assets and Archives are both types of properties, they must be moved out of `resource`. [^1]: For example: https://github.com/pulumi/pulumi/blob/a1d686227cd7e3c70c51bd772450cb0cd57c1479/sdk/go/common/resource/resource_state.go#L35-L36 ## Open Question This PR moves them to their own sub-folders in `resource`. Should `asset` and `archive` live somewhere more high level, like `sdk/go/property/{asset,archive}`? <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> ## Checklist - [ ] I have run `make tidy` to update any new dependencies - [ ] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-01-25 20:39:31 +00:00
case archive.ArchiveSig:
archive, isarchive, err := archive.Deserialize(objmap)
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
if err != nil {
return resource.PropertyValue{}, err
}
contract.Assertf(isarchive, "resource with archive signature is not an archive")
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
return resource.NewArchiveProperty(archive), nil
case resource.SecretSig:
ciphertext, cipherOk := objmap["ciphertext"].(string)
plaintext, plainOk := objmap["plaintext"].(string)
if (!cipherOk && !plainOk) || (plainOk && cipherOk) {
return resource.PropertyValue{}, errors.New(
"malformed secret value: one of `ciphertext` or `plaintext` must be supplied")
}
if plainOk {
encryptedText, err := enc.EncryptValue(ctx, plaintext)
if err != nil {
return resource.PropertyValue{}, fmt.Errorf("encrypting secret value: %w", err)
}
ciphertext = encryptedText
} else {
unencryptedText, err := dec.DecryptValue(ctx, ciphertext)
if err != nil {
return resource.PropertyValue{}, fmt.Errorf("error decrypting secret value: %w", err)
}
plaintext = unencryptedText
}
var elem interface{}
if err := json.Unmarshal([]byte(plaintext), &elem); err != nil {
return resource.PropertyValue{}, err
}
ev, err := DeserializePropertyValue(elem, config.NopDecrypter, enc)
if err != nil {
return resource.PropertyValue{}, err
}
prop := resource.MakeSecret(ev)
// If the decrypter is a cachingCrypter, insert the plain- and ciphertext into the cache with the
// new *resource.Secret as the key.
if cachingCrypter, ok := dec.(*cachingCrypter); ok {
cachingCrypter.insert(prop.SecretValue(), plaintext, ciphertext)
}
return prop, nil
case resource.ResourceReferenceSig:
var packageVersion string
if packageVersionV, ok := objmap["packageVersion"]; ok {
packageVersion, ok = packageVersionV.(string)
if !ok {
return resource.PropertyValue{},
errors.New("malformed resource reference: packageVersion must be a string")
}
}
urnStr, ok := objmap["urn"].(string)
if !ok {
return resource.PropertyValue{}, errors.New("malformed resource reference: missing urn")
}
urn := resource.URN(urnStr)
// deserializeID handles two cases, one of which arose from a bug in a refactoring of resource.ResourceReference.
// This bug caused the raw ID PropertyValue to be serialized as a map[string]interface{}. In the normal case, the
// ID is serialized as a string.
deserializeID := func() (string, bool, error) {
idV, ok := objmap["id"]
if !ok {
return "", false, nil
}
switch idV := idV.(type) {
case string:
return idV, true, nil
case map[string]interface{}:
switch v := idV["V"].(type) {
case nil:
// This happens for component resource references, which do not have an associated ID.
return "", false, nil
case string:
// This happens for custom resource references, which do have an associated ID.
return v, true, nil
case map[string]interface{}:
// This happens for custom resource references with an unknown ID. In this case, the ID should be
// deserialized as the empty string.
return "", true, nil
}
}
return "", false, errors.New("malformed resource reference: id must be a string")
}
id, hasID, err := deserializeID()
if err != nil {
return resource.PropertyValue{}, err
}
if hasID {
return resource.MakeCustomResourceReference(urn, resource.ID(id), packageVersion), nil
}
return resource.MakeComponentResourceReference(urn, packageVersion), nil
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
default:
return resource.PropertyValue{}, fmt.Errorf("unrecognized signature '%v' in property map", sig)
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
}
}
Implement more precise delete-before-replace semantics. (#2369) This implements the new algorithm for deciding which resources must be deleted due to a delete-before-replace operation. We need to compute the set of resources that may be replaced by a change to the resource under consideration. We do this by taking the complete set of transitive dependents on the resource under consideration and removing any resources that would not be replaced by changes to their dependencies. We determine whether or not a resource may be replaced by substituting unknowns for input properties that may change due to deletion of the resources their value depends on and calling the resource provider's Diff method. This is perhaps clearer when described by example. Consider the following dependency graph: A __|__ B C | _|_ D E F In this graph, all of B, C, D, E, and F transitively depend on A. It may be the case, however, that changes to the specific properties of any of those resources R that would occur if a resource on the path to A were deleted and recreated may not cause R to be replaced. For example, the edge from B to A may be a simple dependsOn edge such that a change to B does not actually influence any of B's input properties. In that case, neither B nor D would need to be deleted before A could be deleted. In order to make the above algorithm a reality, the resource monitor interface has been updated to include a map that associates an input property key with the list of resources that input property depends on. Older clients of the resource monitor will leave this map empty, in which case all input properties will be treated as depending on all dependencies of the resource. This is probably overly conservative, but it is less conservative than what we currently implement, and is certainly correct.
2019-01-28 17:46:30 +00:00
// Otherwise, it's just a weakly typed object map.
return resource.NewObjectProperty(obj), nil
default:
contract.Failf("Unrecognized property type %T: %v", v, reflect.ValueOf(v))
}
}
return resource.NewNullProperty(), nil
}