pulumi/pkg/backend/diy/state.go

584 lines
17 KiB
Go
Raw Permalink Normal View History

// Copyright 2016-2022, Pulumi Corporation.
2018-05-22 19:43:36 +00:00
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package diy
import (
"context"
"errors"
"fmt"
"io"
"path"
"path/filepath"
"strings"
"time"
"github.com/pulumi/pulumi/sdk/v3/go/common/env"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/retry"
"github.com/pulumi/pulumi/pkg/v3/engine"
"gocloud.dev/blob"
"gocloud.dev/gcerrors"
"github.com/pulumi/pulumi/pkg/v3/backend"
"github.com/pulumi/pulumi/pkg/v3/resource/deploy"
"github.com/pulumi/pulumi/pkg/v3/resource/stack"
"github.com/pulumi/pulumi/pkg/v3/secrets"
"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
"github.com/pulumi/pulumi/sdk/v3/go/common/encoding"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
)
Improve the overall cloud CLI experience This improves the overall cloud CLI experience workflow. Now whether a stack is local or cloud is inherent to the stack itself. If you interact with a cloud stack, we transparently talk to the cloud; if you interact with a local stack, we just do the right thing, and perform all operations locally. Aside from sometimes seeing a cloud emoji pop-up ☁️, the experience is quite similar. For example, to initialize a new cloud stack, simply: $ pulumi login Logging into Pulumi Cloud: https://pulumi.com/ Enter Pulumi access token: <enter your token> $ pulumi stack init my-cloud-stack Note that you may log into a specific cloud if you'd like. For now, this is just for our own testing purposes, but someday when we support custom clouds (e.g., Enterprise), you can just say: $ pulumi login --cloud-url https://corp.acme.my-ppc.net:9873 The cloud is now the default. If you instead prefer a "fire and forget" style of stack, you can skip the login and pass `--local`: $ pulumi stack init my-faf-stack --local If you are logged in and run `pulumi`, we tell you as much: $ pulumi Usage: pulumi [command] // as before... Currently logged into the Pulumi Cloud ☁️ https://pulumi.com/ And if you list your stacks, we tell you which one is local or not: $ pulumi stack ls NAME LAST UPDATE RESOURCE COUNT CLOUD URL my-cloud-stack 2017-12-01 ... 3 https://pulumi.com/ my-faf-stack n/a 0 n/a And `pulumi stack` by itself prints information like your cloud org, PPC name, and so on, in addition to the usuals. I shall write up more details and make sure to document these changes. This change also fairly significantly refactors the layout of cloud versus local logic, so that the cmd/ package is resonsible for CLI things, and the new pkg/backend/ package is responsible for the backends. The following is the overall resulting package architecture: * The backend.Backend interface can be implemented to substitute a new backend. This has operations to get and list stacks, perform updates, and so on. * The backend.Stack struct is a wrapper around a stack that has or is being manipulated by a Backend. It resembles our existing Stack notions in the engine, but carries additional metadata about its source. Notably, it offers functions that allow operations like updating and deleting on the Backend from which it came. * There is very little else in the pkg/backend/ package. * A new package, pkg/backend/local/, encapsulates all local state management for "fire and forget" scenarios. It simply implements the above logic and contains anything specific to the local experience. * A peer package, pkg/backend/cloud/, encapsulates all logic required for the cloud experience. This includes its subpackage apitype/ which contains JSON schema descriptions required for REST calls against the cloud backend. It also contains handy functions to list which clouds we have authenticated with. * A subpackage here, pkg/backend/state/, is not a provider at all. Instead, it contains all of the state management functions that are currently shared between local and cloud backends. This includes configuration logic -- including encryption -- as well as logic pertaining to which stacks are known to the workspace. This addresses pulumi/pulumi#629 and pulumi/pulumi#494.
2017-12-02 15:29:46 +00:00
// DisableIntegrityChecking can be set to true to disable checkpoint state integrity verification. This is not
// recommended, because it could mean proceeding even in the face of a corrupted checkpoint state file, but can
// be used as a last resort when a command absolutely must be run.
Improve the overall cloud CLI experience This improves the overall cloud CLI experience workflow. Now whether a stack is local or cloud is inherent to the stack itself. If you interact with a cloud stack, we transparently talk to the cloud; if you interact with a local stack, we just do the right thing, and perform all operations locally. Aside from sometimes seeing a cloud emoji pop-up ☁️, the experience is quite similar. For example, to initialize a new cloud stack, simply: $ pulumi login Logging into Pulumi Cloud: https://pulumi.com/ Enter Pulumi access token: <enter your token> $ pulumi stack init my-cloud-stack Note that you may log into a specific cloud if you'd like. For now, this is just for our own testing purposes, but someday when we support custom clouds (e.g., Enterprise), you can just say: $ pulumi login --cloud-url https://corp.acme.my-ppc.net:9873 The cloud is now the default. If you instead prefer a "fire and forget" style of stack, you can skip the login and pass `--local`: $ pulumi stack init my-faf-stack --local If you are logged in and run `pulumi`, we tell you as much: $ pulumi Usage: pulumi [command] // as before... Currently logged into the Pulumi Cloud ☁️ https://pulumi.com/ And if you list your stacks, we tell you which one is local or not: $ pulumi stack ls NAME LAST UPDATE RESOURCE COUNT CLOUD URL my-cloud-stack 2017-12-01 ... 3 https://pulumi.com/ my-faf-stack n/a 0 n/a And `pulumi stack` by itself prints information like your cloud org, PPC name, and so on, in addition to the usuals. I shall write up more details and make sure to document these changes. This change also fairly significantly refactors the layout of cloud versus local logic, so that the cmd/ package is resonsible for CLI things, and the new pkg/backend/ package is responsible for the backends. The following is the overall resulting package architecture: * The backend.Backend interface can be implemented to substitute a new backend. This has operations to get and list stacks, perform updates, and so on. * The backend.Stack struct is a wrapper around a stack that has or is being manipulated by a Backend. It resembles our existing Stack notions in the engine, but carries additional metadata about its source. Notably, it offers functions that allow operations like updating and deleting on the Backend from which it came. * There is very little else in the pkg/backend/ package. * A new package, pkg/backend/local/, encapsulates all local state management for "fire and forget" scenarios. It simply implements the above logic and contains anything specific to the local experience. * A peer package, pkg/backend/cloud/, encapsulates all logic required for the cloud experience. This includes its subpackage apitype/ which contains JSON schema descriptions required for REST calls against the cloud backend. It also contains handy functions to list which clouds we have authenticated with. * A subpackage here, pkg/backend/state/, is not a provider at all. Instead, it contains all of the state management functions that are currently shared between local and cloud backends. This includes configuration logic -- including encryption -- as well as logic pertaining to which stacks are known to the workspace. This addresses pulumi/pulumi#629 and pulumi/pulumi#494.
2017-12-02 15:29:46 +00:00
var DisableIntegrityChecking bool
type diyQuery struct {
root string
proj *workspace.Project
}
func (q *diyQuery) GetRoot() string {
return q.root
}
func (q *diyQuery) GetProject() *workspace.Project {
return q.proj
}
// update is an implementation of engine.Update backed by diy state.
type update struct {
Remove the need to `pulumi init` for the local backend This change removes the need to `pulumi init` when targeting the local backend. A fair amount of the change lays the foundation that the next set of changes to stop having `pulumi init` be used for cloud stacks as well. Previously, `pulumi init` logically did two things: 1. It created the bookkeeping directory for local stacks, this was stored in `<repository-root>/.pulumi`, where `<repository-root>` was the path to what we belived the "root" of your project was. In the case of git repositories, this was the directory that contained your `.git` folder. 2. It recorded repository information in `<repository-root>/.pulumi/repository.json`. This was used by the cloud backend when computing what project to interact with on Pulumi.com The new identity model will remove the need for (2), since we only need an owner and stack name to fully qualify a stack on pulumi.com, so it's easy enough to stop creating a folder just for that. However, for the local backend, we need to continue to retain some information about stacks (e.g. checkpoints, history, etc). In addition, we need to store our workspace settings (which today just contains the selected stack) somehere. For state stored by the local backend, we change the URL scheme from `local://` to `local://<optional-root-path>`. When `<optional-root-path>` is unset, it defaults to `$HOME`. We create our `.pulumi` folder in that directory. This is important because stack names now must be unique within the backend, but we have some tests using local stacks which use fixed stack names, so each integration test really wants its own "view" of the world. For the workspace settings, we introduce a new `workspaces` directory in `~/.pulumi`. In this folder we write the workspace settings file for each project. The file name is the name of the project, combined with the SHA1 of the path of the project file on disk, to ensure that multiple pulumi programs with the same project name have different workspace settings. This does mean that moving a project's location on disk will cause the CLI to "forget" what the selected stack was, which is unfortunate, but not the end of the world. If this ends up being a big pain point, we can certianly try to play games in the future (for example, if we saw a .git folder in a parent folder, we could store data in there). With respect to compatibility, we don't attempt to migrate older files to their newer locations. For long lived stacks managed using the local backend, we can provide information on where to move things to. For all stacks (regardless of backend) we'll require the user to `pulumi stack select` their stack again, but that seems like the correct trade-off vs writing complicated upgrade code.
2018-04-16 23:15:10 +00:00
root string
proj *workspace.Project
target *deploy.Target
backend *diyBackend
}
func (u *update) GetRoot() string {
return u.root
}
func (u *update) GetProject() *workspace.Project {
return u.proj
}
func (u *update) GetTarget() *deploy.Target {
return u.target
}
func (b *diyBackend) newQuery(
ctx context.Context,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
op backend.QueryOperation,
) (engine.QueryInfo, error) {
return &diyQuery{root: op.Root, proj: op.Proj}, nil
}
func (b *diyBackend) newUpdate(
ctx context.Context,
secretsProvider secrets.Provider,
ref *diyBackendReference,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
op backend.UpdateOperation,
) (*update, error) {
contract.Requiref(ref != nil, "ref", "must not be nil")
// Construct the deployment target.
target, err := b.getTarget(ctx, secretsProvider, ref,
op.StackConfiguration.Config, op.StackConfiguration.Decrypter)
if err != nil {
return nil, err
}
// Construct and return a new update.
return &update{
root: op.Root,
proj: op.Proj,
Remove the need to `pulumi init` for the local backend This change removes the need to `pulumi init` when targeting the local backend. A fair amount of the change lays the foundation that the next set of changes to stop having `pulumi init` be used for cloud stacks as well. Previously, `pulumi init` logically did two things: 1. It created the bookkeeping directory for local stacks, this was stored in `<repository-root>/.pulumi`, where `<repository-root>` was the path to what we belived the "root" of your project was. In the case of git repositories, this was the directory that contained your `.git` folder. 2. It recorded repository information in `<repository-root>/.pulumi/repository.json`. This was used by the cloud backend when computing what project to interact with on Pulumi.com The new identity model will remove the need for (2), since we only need an owner and stack name to fully qualify a stack on pulumi.com, so it's easy enough to stop creating a folder just for that. However, for the local backend, we need to continue to retain some information about stacks (e.g. checkpoints, history, etc). In addition, we need to store our workspace settings (which today just contains the selected stack) somehere. For state stored by the local backend, we change the URL scheme from `local://` to `local://<optional-root-path>`. When `<optional-root-path>` is unset, it defaults to `$HOME`. We create our `.pulumi` folder in that directory. This is important because stack names now must be unique within the backend, but we have some tests using local stacks which use fixed stack names, so each integration test really wants its own "view" of the world. For the workspace settings, we introduce a new `workspaces` directory in `~/.pulumi`. In this folder we write the workspace settings file for each project. The file name is the name of the project, combined with the SHA1 of the path of the project file on disk, to ensure that multiple pulumi programs with the same project name have different workspace settings. This does mean that moving a project's location on disk will cause the CLI to "forget" what the selected stack was, which is unfortunate, but not the end of the world. If this ends up being a big pain point, we can certianly try to play games in the future (for example, if we saw a .git folder in a parent folder, we could store data in there). With respect to compatibility, we don't attempt to migrate older files to their newer locations. For long lived stacks managed using the local backend, we can provide information on where to move things to. For all stacks (regardless of backend) we'll require the user to `pulumi stack select` their stack again, but that seems like the correct trade-off vs writing complicated upgrade code.
2018-04-16 23:15:10 +00:00
target: target,
backend: b,
}, nil
}
func (b *diyBackend) getTarget(
ctx context.Context,
secretsProvider secrets.Provider,
ref *diyBackendReference,
cfg config.Map,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
dec config.Decrypter,
) (*deploy.Target, error) {
contract.Requiref(ref != nil, "ref", "must not be nil")
stack, err := b.GetStack(ctx, ref)
if err != nil {
return nil, err
}
snapshot, err := stack.Snapshot(ctx, secretsProvider)
if err != nil {
return nil, err
}
return &deploy.Target{
Name: ref.Name(),
Organization: "organization", // diy has no organizations really, but we just always say it's "organization"
Config: cfg,
Decrypter: dec,
Snapshot: snapshot,
}, nil
}
var errCheckpointNotFound = errors.New("checkpoint does not exist")
// stackExists simply does a check that the checkpoint file we expect for this stack exists.
func (b *diyBackend) stackExists(
ctx context.Context,
ref *diyBackendReference,
) (string, error) {
contract.Requiref(ref != nil, "ref", "must not be nil")
Make some stack-related CLI improvements (#947) This change includes a handful of stack-related CLI formatting improvements that I've been noodling on in the background for a while, based on things that tend to trip up demos and the inner loop workflow. This includes: * If `pulumi stack select` is run by itself, use an interactive CLI menu to let the user select an existing stack, or choose to create a new one. This looks as follows $ pulumi stack select Please choose a stack, or choose to create a new one: abcdef babblabblabble > currentlyselected defcon <create a new stack> and is navigated in the usual way (key up, down, enter). * If a stack name is passed that does not exist, prompt the user to ask whether s/he wants to create one on-demand. This hooks interesting moments in time, like `pulumi stack select foo`, and cuts down on the need to run additional commands. * If a current stack is required, but none is currently selected, then pop the same interactive menu shown above to select one. Depending on the command being run, we may or may not show the option to create a new stack (e.g., that doesn't make much sense when you're running `pulumi destroy`, but might when you're running `pulumi stack`). This again lets you do with a single command what would have otherwise entailed an error with multiple commands to recover from it. * If you run `pulumi stack init` without any additional arguments, we interactively prompt for the stack name. Before, we would error and you'd then need to run `pulumi stack init <name>`. * Colorize some things nicely; for example, now all prompts will by default become bright white.
2018-02-16 23:03:54 +00:00
chkpath := b.stackPath(ctx, ref)
exists, err := b.bucket.Exists(ctx, chkpath)
if err != nil {
return chkpath, fmt.Errorf("failed to load checkpoint: %w", err)
}
if !exists {
return chkpath, errCheckpointNotFound
}
return chkpath, nil
}
func (b *diyBackend) getSnapshot(ctx context.Context,
secretsProvider secrets.Provider, ref *diyBackendReference,
) (*deploy.Snapshot, error) {
contract.Requiref(ref != nil, "ref", "must not be nil")
checkpoint, err := b.getCheckpoint(ctx, ref)
if err != nil {
return nil, fmt.Errorf("failed to load checkpoint: %w", err)
}
Improve the overall cloud CLI experience This improves the overall cloud CLI experience workflow. Now whether a stack is local or cloud is inherent to the stack itself. If you interact with a cloud stack, we transparently talk to the cloud; if you interact with a local stack, we just do the right thing, and perform all operations locally. Aside from sometimes seeing a cloud emoji pop-up ☁️, the experience is quite similar. For example, to initialize a new cloud stack, simply: $ pulumi login Logging into Pulumi Cloud: https://pulumi.com/ Enter Pulumi access token: <enter your token> $ pulumi stack init my-cloud-stack Note that you may log into a specific cloud if you'd like. For now, this is just for our own testing purposes, but someday when we support custom clouds (e.g., Enterprise), you can just say: $ pulumi login --cloud-url https://corp.acme.my-ppc.net:9873 The cloud is now the default. If you instead prefer a "fire and forget" style of stack, you can skip the login and pass `--local`: $ pulumi stack init my-faf-stack --local If you are logged in and run `pulumi`, we tell you as much: $ pulumi Usage: pulumi [command] // as before... Currently logged into the Pulumi Cloud ☁️ https://pulumi.com/ And if you list your stacks, we tell you which one is local or not: $ pulumi stack ls NAME LAST UPDATE RESOURCE COUNT CLOUD URL my-cloud-stack 2017-12-01 ... 3 https://pulumi.com/ my-faf-stack n/a 0 n/a And `pulumi stack` by itself prints information like your cloud org, PPC name, and so on, in addition to the usuals. I shall write up more details and make sure to document these changes. This change also fairly significantly refactors the layout of cloud versus local logic, so that the cmd/ package is resonsible for CLI things, and the new pkg/backend/ package is responsible for the backends. The following is the overall resulting package architecture: * The backend.Backend interface can be implemented to substitute a new backend. This has operations to get and list stacks, perform updates, and so on. * The backend.Stack struct is a wrapper around a stack that has or is being manipulated by a Backend. It resembles our existing Stack notions in the engine, but carries additional metadata about its source. Notably, it offers functions that allow operations like updating and deleting on the Backend from which it came. * There is very little else in the pkg/backend/ package. * A new package, pkg/backend/local/, encapsulates all local state management for "fire and forget" scenarios. It simply implements the above logic and contains anything specific to the local experience. * A peer package, pkg/backend/cloud/, encapsulates all logic required for the cloud experience. This includes its subpackage apitype/ which contains JSON schema descriptions required for REST calls against the cloud backend. It also contains handy functions to list which clouds we have authenticated with. * A subpackage here, pkg/backend/state/, is not a provider at all. Instead, it contains all of the state management functions that are currently shared between local and cloud backends. This includes configuration logic -- including encryption -- as well as logic pertaining to which stacks are known to the workspace. This addresses pulumi/pulumi#629 and pulumi/pulumi#494.
2017-12-02 15:29:46 +00:00
// Materialize an actual snapshot object.
snapshot, err := stack.DeserializeCheckpoint(ctx, secretsProvider, checkpoint)
if err != nil {
return nil, err
}
Improve the overall cloud CLI experience This improves the overall cloud CLI experience workflow. Now whether a stack is local or cloud is inherent to the stack itself. If you interact with a cloud stack, we transparently talk to the cloud; if you interact with a local stack, we just do the right thing, and perform all operations locally. Aside from sometimes seeing a cloud emoji pop-up ☁️, the experience is quite similar. For example, to initialize a new cloud stack, simply: $ pulumi login Logging into Pulumi Cloud: https://pulumi.com/ Enter Pulumi access token: <enter your token> $ pulumi stack init my-cloud-stack Note that you may log into a specific cloud if you'd like. For now, this is just for our own testing purposes, but someday when we support custom clouds (e.g., Enterprise), you can just say: $ pulumi login --cloud-url https://corp.acme.my-ppc.net:9873 The cloud is now the default. If you instead prefer a "fire and forget" style of stack, you can skip the login and pass `--local`: $ pulumi stack init my-faf-stack --local If you are logged in and run `pulumi`, we tell you as much: $ pulumi Usage: pulumi [command] // as before... Currently logged into the Pulumi Cloud ☁️ https://pulumi.com/ And if you list your stacks, we tell you which one is local or not: $ pulumi stack ls NAME LAST UPDATE RESOURCE COUNT CLOUD URL my-cloud-stack 2017-12-01 ... 3 https://pulumi.com/ my-faf-stack n/a 0 n/a And `pulumi stack` by itself prints information like your cloud org, PPC name, and so on, in addition to the usuals. I shall write up more details and make sure to document these changes. This change also fairly significantly refactors the layout of cloud versus local logic, so that the cmd/ package is resonsible for CLI things, and the new pkg/backend/ package is responsible for the backends. The following is the overall resulting package architecture: * The backend.Backend interface can be implemented to substitute a new backend. This has operations to get and list stacks, perform updates, and so on. * The backend.Stack struct is a wrapper around a stack that has or is being manipulated by a Backend. It resembles our existing Stack notions in the engine, but carries additional metadata about its source. Notably, it offers functions that allow operations like updating and deleting on the Backend from which it came. * There is very little else in the pkg/backend/ package. * A new package, pkg/backend/local/, encapsulates all local state management for "fire and forget" scenarios. It simply implements the above logic and contains anything specific to the local experience. * A peer package, pkg/backend/cloud/, encapsulates all logic required for the cloud experience. This includes its subpackage apitype/ which contains JSON schema descriptions required for REST calls against the cloud backend. It also contains handy functions to list which clouds we have authenticated with. * A subpackage here, pkg/backend/state/, is not a provider at all. Instead, it contains all of the state management functions that are currently shared between local and cloud backends. This includes configuration logic -- including encryption -- as well as logic pertaining to which stacks are known to the workspace. This addresses pulumi/pulumi#629 and pulumi/pulumi#494.
2017-12-02 15:29:46 +00:00
// Ensure the snapshot passes verification before returning it, to catch bugs early.
Validate snapshots from service on load (#14046) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> In the filestate code we would always run loaded snapshots through VerifyIntegrity before using them. The httpstate didn't have any similar checks. This brings the two backends into alignment, reusing the option from before that was just for filestate (--disable-integrity-checking). At some point we should further align these so that httpstate also validates the snapshots it has written out, like filestate does today. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-04 15:12:56 +00:00
if !backend.DisableIntegrityChecking {
if err := snapshot.VerifyIntegrity(); err != nil {
return nil, fmt.Errorf("snapshot integrity failure; refusing to use it: %w", err)
}
}
return snapshot, nil
}
Remove the need to `pulumi init` for the local backend This change removes the need to `pulumi init` when targeting the local backend. A fair amount of the change lays the foundation that the next set of changes to stop having `pulumi init` be used for cloud stacks as well. Previously, `pulumi init` logically did two things: 1. It created the bookkeeping directory for local stacks, this was stored in `<repository-root>/.pulumi`, where `<repository-root>` was the path to what we belived the "root" of your project was. In the case of git repositories, this was the directory that contained your `.git` folder. 2. It recorded repository information in `<repository-root>/.pulumi/repository.json`. This was used by the cloud backend when computing what project to interact with on Pulumi.com The new identity model will remove the need for (2), since we only need an owner and stack name to fully qualify a stack on pulumi.com, so it's easy enough to stop creating a folder just for that. However, for the local backend, we need to continue to retain some information about stacks (e.g. checkpoints, history, etc). In addition, we need to store our workspace settings (which today just contains the selected stack) somehere. For state stored by the local backend, we change the URL scheme from `local://` to `local://<optional-root-path>`. When `<optional-root-path>` is unset, it defaults to `$HOME`. We create our `.pulumi` folder in that directory. This is important because stack names now must be unique within the backend, but we have some tests using local stacks which use fixed stack names, so each integration test really wants its own "view" of the world. For the workspace settings, we introduce a new `workspaces` directory in `~/.pulumi`. In this folder we write the workspace settings file for each project. The file name is the name of the project, combined with the SHA1 of the path of the project file on disk, to ensure that multiple pulumi programs with the same project name have different workspace settings. This does mean that moving a project's location on disk will cause the CLI to "forget" what the selected stack was, which is unfortunate, but not the end of the world. If this ends up being a big pain point, we can certianly try to play games in the future (for example, if we saw a .git folder in a parent folder, we could store data in there). With respect to compatibility, we don't attempt to migrate older files to their newer locations. For long lived stacks managed using the local backend, we can provide information on where to move things to. For all stacks (regardless of backend) we'll require the user to `pulumi stack select` their stack again, but that seems like the correct trade-off vs writing complicated upgrade code.
2018-04-16 23:15:10 +00:00
// GetCheckpoint loads a checkpoint file for the given stack in this project, from the current project workspace.
func (b *diyBackend) getCheckpoint(ctx context.Context, ref *diyBackendReference) (*apitype.CheckpointV3, error) {
chkpath := b.stackPath(ctx, ref)
bytes, err := b.bucket.ReadAll(ctx, chkpath)
if err != nil {
Remove the need to `pulumi init` for the local backend This change removes the need to `pulumi init` when targeting the local backend. A fair amount of the change lays the foundation that the next set of changes to stop having `pulumi init` be used for cloud stacks as well. Previously, `pulumi init` logically did two things: 1. It created the bookkeeping directory for local stacks, this was stored in `<repository-root>/.pulumi`, where `<repository-root>` was the path to what we belived the "root" of your project was. In the case of git repositories, this was the directory that contained your `.git` folder. 2. It recorded repository information in `<repository-root>/.pulumi/repository.json`. This was used by the cloud backend when computing what project to interact with on Pulumi.com The new identity model will remove the need for (2), since we only need an owner and stack name to fully qualify a stack on pulumi.com, so it's easy enough to stop creating a folder just for that. However, for the local backend, we need to continue to retain some information about stacks (e.g. checkpoints, history, etc). In addition, we need to store our workspace settings (which today just contains the selected stack) somehere. For state stored by the local backend, we change the URL scheme from `local://` to `local://<optional-root-path>`. When `<optional-root-path>` is unset, it defaults to `$HOME`. We create our `.pulumi` folder in that directory. This is important because stack names now must be unique within the backend, but we have some tests using local stacks which use fixed stack names, so each integration test really wants its own "view" of the world. For the workspace settings, we introduce a new `workspaces` directory in `~/.pulumi`. In this folder we write the workspace settings file for each project. The file name is the name of the project, combined with the SHA1 of the path of the project file on disk, to ensure that multiple pulumi programs with the same project name have different workspace settings. This does mean that moving a project's location on disk will cause the CLI to "forget" what the selected stack was, which is unfortunate, but not the end of the world. If this ends up being a big pain point, we can certianly try to play games in the future (for example, if we saw a .git folder in a parent folder, we could store data in there). With respect to compatibility, we don't attempt to migrate older files to their newer locations. For long lived stacks managed using the local backend, we can provide information on where to move things to. For all stacks (regardless of backend) we'll require the user to `pulumi stack select` their stack again, but that seems like the correct trade-off vs writing complicated upgrade code.
2018-04-16 23:15:10 +00:00
return nil, err
}
m := encoding.JSON
if encoding.IsCompressed(bytes) {
m = encoding.Gzip(m)
}
return stack.UnmarshalVersionedCheckpointToLatestCheckpoint(m, bytes)
Remove the need to `pulumi init` for the local backend This change removes the need to `pulumi init` when targeting the local backend. A fair amount of the change lays the foundation that the next set of changes to stop having `pulumi init` be used for cloud stacks as well. Previously, `pulumi init` logically did two things: 1. It created the bookkeeping directory for local stacks, this was stored in `<repository-root>/.pulumi`, where `<repository-root>` was the path to what we belived the "root" of your project was. In the case of git repositories, this was the directory that contained your `.git` folder. 2. It recorded repository information in `<repository-root>/.pulumi/repository.json`. This was used by the cloud backend when computing what project to interact with on Pulumi.com The new identity model will remove the need for (2), since we only need an owner and stack name to fully qualify a stack on pulumi.com, so it's easy enough to stop creating a folder just for that. However, for the local backend, we need to continue to retain some information about stacks (e.g. checkpoints, history, etc). In addition, we need to store our workspace settings (which today just contains the selected stack) somehere. For state stored by the local backend, we change the URL scheme from `local://` to `local://<optional-root-path>`. When `<optional-root-path>` is unset, it defaults to `$HOME`. We create our `.pulumi` folder in that directory. This is important because stack names now must be unique within the backend, but we have some tests using local stacks which use fixed stack names, so each integration test really wants its own "view" of the world. For the workspace settings, we introduce a new `workspaces` directory in `~/.pulumi`. In this folder we write the workspace settings file for each project. The file name is the name of the project, combined with the SHA1 of the path of the project file on disk, to ensure that multiple pulumi programs with the same project name have different workspace settings. This does mean that moving a project's location on disk will cause the CLI to "forget" what the selected stack was, which is unfortunate, but not the end of the world. If this ends up being a big pain point, we can certianly try to play games in the future (for example, if we saw a .git folder in a parent folder, we could store data in there). With respect to compatibility, we don't attempt to migrate older files to their newer locations. For long lived stacks managed using the local backend, we can provide information on where to move things to. For all stacks (regardless of backend) we'll require the user to `pulumi stack select` their stack again, but that seems like the correct trade-off vs writing complicated upgrade code.
2018-04-16 23:15:10 +00:00
}
func (b *diyBackend) saveCheckpoint(
ctx context.Context,
ref *diyBackendReference,
checkpoint *apitype.VersionedCheckpoint,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
) (backupFile string, file string, _ error) {
// Make a serializable stack and then use the encoder to encode it.
file = b.stackPath(ctx, ref)
m, ext := encoding.Detect(strings.TrimSuffix(file, ".gz"))
if m == nil {
return "", "", fmt.Errorf("resource serialization failed; illegal markup extension: '%v'", ext)
}
if filepath.Ext(file) == "" {
file = file + ext
}
if b.gzip {
if filepath.Ext(file) != encoding.GZIPExt {
file = file + ".gz"
}
m = encoding.Gzip(m)
} else {
file = strings.TrimSuffix(file, ".gz")
}
byts, err := m.Marshal(checkpoint)
if err != nil {
return "", "", fmt.Errorf("An IO error occurred while marshalling the checkpoint: %w", err)
}
// Back up the existing file if it already exists. Don't delete the original, the following WriteAll will
// atomically replace it anyway and various other bits of the system depend on being able to find the
// .json file to know the stack currently exists (see https://github.com/pulumi/pulumi/issues/9033 for
// context).
filePlain := strings.TrimSuffix(file, ".gz")
fileGzip := filePlain + ".gz"
// We need to make sure that an out of date state file doesn't exist so we
// only keep the file of the type we are working with.
bckGzip := backupTarget(ctx, b.bucket, fileGzip, b.gzip)
bckPlain := backupTarget(ctx, b.bucket, filePlain, !b.gzip)
if b.gzip {
backupFile = bckGzip
} else {
backupFile = bckPlain
}
// And now write out the new snapshot file, overwriting that location.
if err = b.bucket.WriteAll(ctx, file, byts, nil); err != nil {
b.mutex.Lock()
defer b.mutex.Unlock()
2020-04-28 00:14:07 +00:00
// FIXME: Would be nice to make these configurable
delay, _ := time.ParseDuration("1s")
maxDelay, _ := time.ParseDuration("30s")
backoff := 1.2
// Retry the write 10 times in case of upstream bucket errors
_, _, err = retry.Until(ctx, retry.Acceptor{
2020-04-28 00:23:47 +00:00
Delay: &delay,
2020-04-28 00:14:07 +00:00
MaxDelay: &maxDelay,
2020-04-28 00:23:47 +00:00
Backoff: &backoff,
Accept: func(try int, nextRetryTime time.Duration) (bool, interface{}, error) {
// And now write out the new snapshot file, overwriting that location.
err := b.bucket.WriteAll(ctx, file, byts, nil)
if err != nil {
logging.V(7).Infof("Error while writing snapshot to: %s (attempt=%d, error=%s)", file, try, err)
if try > 10 {
return false, nil, fmt.Errorf("An IO error occurred while writing the new snapshot file: %w", err)
}
return false, nil, nil
}
return true, nil, nil
},
})
if err != nil {
return backupFile, "", err
}
}
logging.V(7).Infof("Saved stack %s checkpoint to: %s (backup=%s)", ref.FullyQualifiedName(), file, backupFile)
// And if we are retaining historical checkpoint information, write it out again
if b.Env.GetBool(env.DIYBackendRetainCheckpoints) {
if err = b.bucket.WriteAll(ctx, fmt.Sprintf("%v.%v", file, time.Now().UnixNano()), byts, nil); err != nil {
return backupFile, "", fmt.Errorf("An IO error occurred while writing the new snapshot file: %w", err)
}
}
return backupFile, file, nil
}
func (b *diyBackend) saveStack(
ctx context.Context,
ref *diyBackendReference, snap *deploy.Snapshot,
sm secrets.Manager,
) (string, error) {
contract.Requiref(ref != nil, "ref", "ref was nil")
Always use the snapshot secret manager (#15768) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> There were a number of places where we passed a `Snapshot` and a `secret.Manager` as arguments to a method, where if the `Manger` was nil we'd fall back to the `Snapshot.SecretManager` (which could also be nil). Turns out in all but one place this was always passed as nil or just as directly the snapshot's `SecretManager` field. The one place it differed was in `pkg/cmd/pulumi/stack_change_secrets_provider.go` where we're changing the secret manager, but it's fine to just set the snapshot's `SecretManager` field to the new manager. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-03-25 10:30:14 +00:00
chk, err := stack.SerializeCheckpoint(ref.FullyQualifiedName(), snap, false /* showSecrets */)
if err != nil {
return "", fmt.Errorf("serializaing checkpoint: %w", err)
}
backup, file, err := b.saveCheckpoint(ctx, ref, chk)
if err != nil {
return "", err
}
Validate snapshots from service on load (#14046) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> In the filestate code we would always run loaded snapshots through VerifyIntegrity before using them. The httpstate didn't have any similar checks. This brings the two backends into alignment, reusing the option from before that was just for filestate (--disable-integrity-checking). At some point we should further align these so that httpstate also validates the snapshots it has written out, like filestate does today. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-04 15:12:56 +00:00
if !backend.DisableIntegrityChecking {
// Finally, *after* writing the checkpoint, check the integrity. This is done afterwards so that we write
// out the checkpoint file since it may contain resource state updates. But we will warn the user that the
// file is already written and might be bad.
if verifyerr := snap.VerifyIntegrity(); verifyerr != nil {
return "", fmt.Errorf(
"%s: snapshot integrity failure; it was already written, but is invalid (backup available at %s): %w",
file, backup, verifyerr)
}
}
Make some stack-related CLI improvements (#947) This change includes a handful of stack-related CLI formatting improvements that I've been noodling on in the background for a while, based on things that tend to trip up demos and the inner loop workflow. This includes: * If `pulumi stack select` is run by itself, use an interactive CLI menu to let the user select an existing stack, or choose to create a new one. This looks as follows $ pulumi stack select Please choose a stack, or choose to create a new one: abcdef babblabblabble > currentlyselected defcon <create a new stack> and is navigated in the usual way (key up, down, enter). * If a stack name is passed that does not exist, prompt the user to ask whether s/he wants to create one on-demand. This hooks interesting moments in time, like `pulumi stack select foo`, and cuts down on the need to run additional commands. * If a current stack is required, but none is currently selected, then pop the same interactive menu shown above to select one. Depending on the command being run, we may or may not show the option to create a new stack (e.g., that doesn't make much sense when you're running `pulumi destroy`, but might when you're running `pulumi stack`). This again lets you do with a single command what would have otherwise entailed an error with multiple commands to recover from it. * If you run `pulumi stack init` without any additional arguments, we interactively prompt for the stack name. Before, we would error and you'd then need to run `pulumi stack init <name>`. * Colorize some things nicely; for example, now all prompts will by default become bright white.
2018-02-16 23:03:54 +00:00
return file, nil
}
Improve the overall cloud CLI experience This improves the overall cloud CLI experience workflow. Now whether a stack is local or cloud is inherent to the stack itself. If you interact with a cloud stack, we transparently talk to the cloud; if you interact with a local stack, we just do the right thing, and perform all operations locally. Aside from sometimes seeing a cloud emoji pop-up ☁️, the experience is quite similar. For example, to initialize a new cloud stack, simply: $ pulumi login Logging into Pulumi Cloud: https://pulumi.com/ Enter Pulumi access token: <enter your token> $ pulumi stack init my-cloud-stack Note that you may log into a specific cloud if you'd like. For now, this is just for our own testing purposes, but someday when we support custom clouds (e.g., Enterprise), you can just say: $ pulumi login --cloud-url https://corp.acme.my-ppc.net:9873 The cloud is now the default. If you instead prefer a "fire and forget" style of stack, you can skip the login and pass `--local`: $ pulumi stack init my-faf-stack --local If you are logged in and run `pulumi`, we tell you as much: $ pulumi Usage: pulumi [command] // as before... Currently logged into the Pulumi Cloud ☁️ https://pulumi.com/ And if you list your stacks, we tell you which one is local or not: $ pulumi stack ls NAME LAST UPDATE RESOURCE COUNT CLOUD URL my-cloud-stack 2017-12-01 ... 3 https://pulumi.com/ my-faf-stack n/a 0 n/a And `pulumi stack` by itself prints information like your cloud org, PPC name, and so on, in addition to the usuals. I shall write up more details and make sure to document these changes. This change also fairly significantly refactors the layout of cloud versus local logic, so that the cmd/ package is resonsible for CLI things, and the new pkg/backend/ package is responsible for the backends. The following is the overall resulting package architecture: * The backend.Backend interface can be implemented to substitute a new backend. This has operations to get and list stacks, perform updates, and so on. * The backend.Stack struct is a wrapper around a stack that has or is being manipulated by a Backend. It resembles our existing Stack notions in the engine, but carries additional metadata about its source. Notably, it offers functions that allow operations like updating and deleting on the Backend from which it came. * There is very little else in the pkg/backend/ package. * A new package, pkg/backend/local/, encapsulates all local state management for "fire and forget" scenarios. It simply implements the above logic and contains anything specific to the local experience. * A peer package, pkg/backend/cloud/, encapsulates all logic required for the cloud experience. This includes its subpackage apitype/ which contains JSON schema descriptions required for REST calls against the cloud backend. It also contains handy functions to list which clouds we have authenticated with. * A subpackage here, pkg/backend/state/, is not a provider at all. Instead, it contains all of the state management functions that are currently shared between local and cloud backends. This includes configuration logic -- including encryption -- as well as logic pertaining to which stacks are known to the workspace. This addresses pulumi/pulumi#629 and pulumi/pulumi#494.
2017-12-02 15:29:46 +00:00
// removeStack removes information about a stack from the current workspace.
func (b *diyBackend) removeStack(ctx context.Context, ref *diyBackendReference) error {
contract.Requiref(ref != nil, "ref", "must not be nil")
// Just make a backup of the file and don't write out anything new.
file := b.stackPath(ctx, ref)
backupTarget(ctx, b.bucket, file, false)
historyDir := ref.HistoryDir()
return removeAllByPrefix(ctx, b.bucket, historyDir)
}
// backupTarget makes a backup of an existing file, in preparation for writing a new one.
func backupTarget(ctx context.Context, bucket Bucket, file string, keepOriginal bool) string {
contract.Requiref(file != "", "file", "must not be empty")
bck := file + ".bak"
err := bucket.Copy(ctx, bck, file, nil)
if err != nil {
logging.V(5).Infof("error copying %s to %s: %s", file, bck, err)
}
if !keepOriginal {
err = bucket.Delete(ctx, file)
if err != nil {
logging.V(5).Infof("error deleting source object after rename: %v (%v) skipping", file, err)
}
}
// IDEA: consider multiple backups (.bak.bak.bak...etc).
return bck
}
// backupStack copies the current Checkpoint file to ~/.pulumi/backups.
func (b *diyBackend) backupStack(ctx context.Context, ref *diyBackendReference) error {
contract.Requiref(ref != nil, "ref", "must not be nil")
// Exit early if backups are disabled.
if b.Env.GetBool(env.DIYBackendDisableCheckpointBackups) {
return nil
}
// Read the current checkpoint file. (Assuming it aleady exists.)
stackPath := b.stackPath(ctx, ref)
byts, err := b.bucket.ReadAll(ctx, stackPath)
if err != nil {
return err
}
// Get the backup directory.
backupDir := ref.BackupDir()
// Write out the new backup checkpoint file.
stackFile := filepath.Base(stackPath)
ext := filepath.Ext(stackFile)
base := strings.TrimSuffix(stackFile, ext)
if ext2 := filepath.Ext(base); ext2 != "" && ext == encoding.GZIPExt {
// base: stack-name.json, ext: .gz
// ->
// base: stack-name, ext: .json.gz
ext = ext2 + ext
base = strings.TrimSuffix(base, ext2)
}
backupFile := fmt.Sprintf("%s.%v%s", base, time.Now().UnixNano(), ext)
return b.bucket.WriteAll(ctx, filepath.Join(backupDir, backupFile), byts, nil)
Remove the need to `pulumi init` for the local backend This change removes the need to `pulumi init` when targeting the local backend. A fair amount of the change lays the foundation that the next set of changes to stop having `pulumi init` be used for cloud stacks as well. Previously, `pulumi init` logically did two things: 1. It created the bookkeeping directory for local stacks, this was stored in `<repository-root>/.pulumi`, where `<repository-root>` was the path to what we belived the "root" of your project was. In the case of git repositories, this was the directory that contained your `.git` folder. 2. It recorded repository information in `<repository-root>/.pulumi/repository.json`. This was used by the cloud backend when computing what project to interact with on Pulumi.com The new identity model will remove the need for (2), since we only need an owner and stack name to fully qualify a stack on pulumi.com, so it's easy enough to stop creating a folder just for that. However, for the local backend, we need to continue to retain some information about stacks (e.g. checkpoints, history, etc). In addition, we need to store our workspace settings (which today just contains the selected stack) somehere. For state stored by the local backend, we change the URL scheme from `local://` to `local://<optional-root-path>`. When `<optional-root-path>` is unset, it defaults to `$HOME`. We create our `.pulumi` folder in that directory. This is important because stack names now must be unique within the backend, but we have some tests using local stacks which use fixed stack names, so each integration test really wants its own "view" of the world. For the workspace settings, we introduce a new `workspaces` directory in `~/.pulumi`. In this folder we write the workspace settings file for each project. The file name is the name of the project, combined with the SHA1 of the path of the project file on disk, to ensure that multiple pulumi programs with the same project name have different workspace settings. This does mean that moving a project's location on disk will cause the CLI to "forget" what the selected stack was, which is unfortunate, but not the end of the world. If this ends up being a big pain point, we can certianly try to play games in the future (for example, if we saw a .git folder in a parent folder, we could store data in there). With respect to compatibility, we don't attempt to migrate older files to their newer locations. For long lived stacks managed using the local backend, we can provide information on where to move things to. For all stacks (regardless of backend) we'll require the user to `pulumi stack select` their stack again, but that seems like the correct trade-off vs writing complicated upgrade code.
2018-04-16 23:15:10 +00:00
}
func (b *diyBackend) stackPath(ctx context.Context, ref *diyBackendReference) string {
if ref == nil {
return StacksDir
}
// We can't use listBucket here for as we need to do a partial prefix match on filename, while the
// "dir" option to listBucket is always suffixed with "/". Also means we don't need to save any
// results in a slice.
plainPath := filepath.ToSlash(ref.StackBasePath()) + ".json"
gzipedPath := plainPath + ".gz"
bucketIter := b.bucket.List(&blob.ListOptions{
Delimiter: "/",
Prefix: plainPath,
})
var plainObj *blob.ListObject
for {
file, err := bucketIter.Next(ctx)
if err == io.EOF {
break
}
if err != nil {
// Error fetching the available ojects, assume .json
return plainPath
}
// plainObj will always come out first since allObjs is sorted by Key
if file.Key == plainPath {
plainObj = file
} else if file.Key == gzipedPath {
// We have a plain .json file and it was modified after this gzipped one so use it.
if plainObj != nil && plainObj.ModTime.After(file.ModTime) {
return plainPath
}
// else use the gzipped object
return gzipedPath
}
Remove the need to `pulumi init` for the local backend This change removes the need to `pulumi init` when targeting the local backend. A fair amount of the change lays the foundation that the next set of changes to stop having `pulumi init` be used for cloud stacks as well. Previously, `pulumi init` logically did two things: 1. It created the bookkeeping directory for local stacks, this was stored in `<repository-root>/.pulumi`, where `<repository-root>` was the path to what we belived the "root" of your project was. In the case of git repositories, this was the directory that contained your `.git` folder. 2. It recorded repository information in `<repository-root>/.pulumi/repository.json`. This was used by the cloud backend when computing what project to interact with on Pulumi.com The new identity model will remove the need for (2), since we only need an owner and stack name to fully qualify a stack on pulumi.com, so it's easy enough to stop creating a folder just for that. However, for the local backend, we need to continue to retain some information about stacks (e.g. checkpoints, history, etc). In addition, we need to store our workspace settings (which today just contains the selected stack) somehere. For state stored by the local backend, we change the URL scheme from `local://` to `local://<optional-root-path>`. When `<optional-root-path>` is unset, it defaults to `$HOME`. We create our `.pulumi` folder in that directory. This is important because stack names now must be unique within the backend, but we have some tests using local stacks which use fixed stack names, so each integration test really wants its own "view" of the world. For the workspace settings, we introduce a new `workspaces` directory in `~/.pulumi`. In this folder we write the workspace settings file for each project. The file name is the name of the project, combined with the SHA1 of the path of the project file on disk, to ensure that multiple pulumi programs with the same project name have different workspace settings. This does mean that moving a project's location on disk will cause the CLI to "forget" what the selected stack was, which is unfortunate, but not the end of the world. If this ends up being a big pain point, we can certianly try to play games in the future (for example, if we saw a .git folder in a parent folder, we could store data in there). With respect to compatibility, we don't attempt to migrate older files to their newer locations. For long lived stacks managed using the local backend, we can provide information on where to move things to. For all stacks (regardless of backend) we'll require the user to `pulumi stack select` their stack again, but that seems like the correct trade-off vs writing complicated upgrade code.
2018-04-16 23:15:10 +00:00
}
// Couldn't find any objects, assume nongzipped path?
return plainPath
Remove the need to `pulumi init` for the local backend This change removes the need to `pulumi init` when targeting the local backend. A fair amount of the change lays the foundation that the next set of changes to stop having `pulumi init` be used for cloud stacks as well. Previously, `pulumi init` logically did two things: 1. It created the bookkeeping directory for local stacks, this was stored in `<repository-root>/.pulumi`, where `<repository-root>` was the path to what we belived the "root" of your project was. In the case of git repositories, this was the directory that contained your `.git` folder. 2. It recorded repository information in `<repository-root>/.pulumi/repository.json`. This was used by the cloud backend when computing what project to interact with on Pulumi.com The new identity model will remove the need for (2), since we only need an owner and stack name to fully qualify a stack on pulumi.com, so it's easy enough to stop creating a folder just for that. However, for the local backend, we need to continue to retain some information about stacks (e.g. checkpoints, history, etc). In addition, we need to store our workspace settings (which today just contains the selected stack) somehere. For state stored by the local backend, we change the URL scheme from `local://` to `local://<optional-root-path>`. When `<optional-root-path>` is unset, it defaults to `$HOME`. We create our `.pulumi` folder in that directory. This is important because stack names now must be unique within the backend, but we have some tests using local stacks which use fixed stack names, so each integration test really wants its own "view" of the world. For the workspace settings, we introduce a new `workspaces` directory in `~/.pulumi`. In this folder we write the workspace settings file for each project. The file name is the name of the project, combined with the SHA1 of the path of the project file on disk, to ensure that multiple pulumi programs with the same project name have different workspace settings. This does mean that moving a project's location on disk will cause the CLI to "forget" what the selected stack was, which is unfortunate, but not the end of the world. If this ends up being a big pain point, we can certianly try to play games in the future (for example, if we saw a .git folder in a parent folder, we could store data in there). With respect to compatibility, we don't attempt to migrate older files to their newer locations. For long lived stacks managed using the local backend, we can provide information on where to move things to. For all stacks (regardless of backend) we'll require the user to `pulumi stack select` their stack again, but that seems like the correct trade-off vs writing complicated upgrade code.
2018-04-16 23:15:10 +00:00
}
// getHistory returns stored update history. The first element of the result will be
// the most recent update record.
func (b *diyBackend) getHistory(
ctx context.Context,
stack *diyBackendReference,
pageSize int, page int,
) ([]backend.UpdateInfo, error) {
contract.Requiref(stack != nil, "stack", "must not be nil")
dir := stack.HistoryDir()
// TODO: we could consider optimizing the list operation using `page` and `pageSize`.
// Unfortunately, this is mildly invasive given the gocloud List API.
allFiles, err := listBucket(ctx, b.bucket, dir)
if err != nil {
// History doesn't exist until a stack has been updated.
if gcerrors.Code(err) == gcerrors.NotFound {
return nil, nil
}
return nil, err
}
var historyEntries []*blob.ListObject
// filter down to just history entries, reversing list to be in most recent order.
// listBucket returns the array sorted by file name, but because of how we name files, older updates come before
// newer ones.
for i := len(allFiles) - 1; i >= 0; i-- {
file := allFiles[i]
filepath := file.Key
// ignore checkpoints
if !strings.HasSuffix(filepath, ".history.json") &&
!strings.HasSuffix(filepath, ".history.json.gz") {
continue
}
historyEntries = append(historyEntries, file)
}
start := 0
end := len(historyEntries) - 1
if pageSize > 0 {
if page < 1 {
page = 1
}
start = (page - 1) * pageSize
end = start + pageSize - 1
if end > len(historyEntries)-1 {
end = len(historyEntries) - 1
}
}
var updates []backend.UpdateInfo
for i := start; i <= end; i++ {
file := historyEntries[i]
filepath := file.Key
var update backend.UpdateInfo
b, err := b.bucket.ReadAll(ctx, filepath)
if err != nil {
return nil, fmt.Errorf("reading history file %s: %w", filepath, err)
}
m := encoding.JSON
if encoding.IsCompressed(b) {
m = encoding.Gzip(m)
}
err = m.Unmarshal(b, &update)
if err != nil {
return nil, fmt.Errorf("reading history file %s: %w", filepath, err)
}
updates = append(updates, update)
}
return updates, nil
}
func (b *diyBackend) renameHistory(ctx context.Context, oldName, newName *diyBackendReference) error {
contract.Requiref(oldName != nil, "oldName", "must not be nil")
contract.Requiref(newName != nil, "newName", "must not be nil")
oldHistory := oldName.HistoryDir()
newHistory := newName.HistoryDir()
allFiles, err := listBucket(ctx, b.bucket, oldHistory)
if err != nil {
// if there's nothing there, we don't really need to do a rename.
if gcerrors.Code(err) == gcerrors.NotFound {
return nil
}
return err
}
for _, file := range allFiles {
fileName := objectName(file)
oldBlob := path.Join(oldHistory, fileName)
// The filename format is <stack-name>-<timestamp>.[checkpoint|history].json[.gz], we need to change
// the stack name part but retain the other parts. If we find files that don't match this format
// ignore them.
dashIndex := strings.LastIndex(fileName, "-")
if dashIndex == -1 || (fileName[:dashIndex] != oldName.name.String()) {
// No dash or the string up to the dash isn't the old name
continue
}
newFileName := newName.name.String() + fileName[dashIndex:]
newBlob := path.Join(newHistory, newFileName)
if err := b.bucket.Copy(ctx, newBlob, oldBlob, nil); err != nil {
return fmt.Errorf("copying history file: %w", err)
}
if err := b.bucket.Delete(ctx, oldBlob); err != nil {
return fmt.Errorf("deleting existing history file: %w", err)
}
}
return nil
}
// addToHistory saves the UpdateInfo and makes a copy of the current Checkpoint file.
func (b *diyBackend) addToHistory(ctx context.Context, ref *diyBackendReference, update backend.UpdateInfo) error {
contract.Requiref(ref != nil, "ref", "must not be nil")
dir := ref.HistoryDir()
// Prefix for the update and checkpoint files.
pathPrefix := path.Join(dir, fmt.Sprintf("%s-%d", ref.name, time.Now().UnixNano()))
m, ext := encoding.JSON, "json"
if b.gzip {
m = encoding.Gzip(m)
ext += ".gz"
}
// Save the history file.
byts, err := m.Marshal(&update)
if err != nil {
return err
}
historyFile := fmt.Sprintf("%s.history.%s", pathPrefix, ext)
if err = b.bucket.WriteAll(ctx, historyFile, byts, nil); err != nil {
return err
}
// Make a copy of the checkpoint file. (Assuming it already exists.)
checkpointFile := fmt.Sprintf("%s.checkpoint.%s", pathPrefix, ext)
return b.bucket.Copy(ctx, checkpointFile, b.stackPath(ctx, ref), nil)
}