pulumi/pkg/engine/lifecycletest/pending_delete_test.go

172 lines
4.7 KiB
Go
Raw Permalink Normal View History

package lifecycletest
import (
"testing"
"github.com/blang/semver"
"github.com/stretchr/testify/assert"
2023-11-21 15:16:13 +00:00
. "github.com/pulumi/pulumi/pkg/v3/engine" //nolint:revive
"github.com/pulumi/pulumi/pkg/v3/resource/deploy"
"github.com/pulumi/pulumi/pkg/v3/resource/deploy/deploytest"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
)
func TestDestroyWithPendingDelete(t *testing.T) {
t.Parallel()
loaders := []*deploytest.ProviderLoader{
deploytest.NewProviderLoader("pkgA", semver.MustParse("1.0.0"), func() (plugin.Provider, error) {
return &deploytest.Provider{}, nil
}),
}
Lifecycle tests shouldn't use a closed host (#14063) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This PR fixes the inadvertent use of a closed plugin host in the lifecycle tests. The tests override the host that is provided to the engine, for good reasons, but that same host is re-used across multiple engine operations. Since the engine closes the supplied host at the end of each operation, subsequent operations are handed a closed host. In order to detect engine bugs related to the use of a closed host (see https://github.com/pulumi/pulumi/pull/14057), the fake host should return an error if it is used after being closed (as does the real host). This PR addresses this. The detailed change is to shift to using a host factory that produces a host in `TestOp.Run`. The `TestPlan` now takes a `TestUpdateOptions` with `HostF` and an embedded `UpdateOptions`. Note that two tests fail due to https://github.com/pulumi/pulumi/pull/14057 which was being masked by the problem that is fixed here. This PR disables those tests and the other PR will re-enable them. - `TestCanceledRefresh` - `TestProviderCancellation` ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-28 21:50:18 +00:00
programF := deploytest.NewLanguageRuntimeF(func(_ plugin.RunInfo, _ *deploytest.ResourceMonitor) error {
return nil
})
Lifecycle tests shouldn't use a closed host (#14063) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This PR fixes the inadvertent use of a closed plugin host in the lifecycle tests. The tests override the host that is provided to the engine, for good reasons, but that same host is re-used across multiple engine operations. Since the engine closes the supplied host at the end of each operation, subsequent operations are handed a closed host. In order to detect engine bugs related to the use of a closed host (see https://github.com/pulumi/pulumi/pull/14057), the fake host should return an error if it is used after being closed (as does the real host). This PR addresses this. The detailed change is to shift to using a host factory that produces a host in `TestOp.Run`. The `TestPlan` now takes a `TestUpdateOptions` with `HostF` and an embedded `UpdateOptions`. Note that two tests fail due to https://github.com/pulumi/pulumi/pull/14057 which was being masked by the problem that is fixed here. This PR disables those tests and the other PR will re-enable them. - `TestCanceledRefresh` - `TestProviderCancellation` ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-28 21:50:18 +00:00
hostF := deploytest.NewPluginHostF(nil, nil, programF, loaders...)
p := &TestPlan{
Add display to the engine tests (#16050) We want to add more test coverage to the display code. The best way to do that is to add it to the engine tests, that already cover most of the pulumi functionality. It's probably not really possible to review all of the output, but at least it gives us a baseline, which we can work with. There's a couple of tests that are flaky for reasons I don't quite understand yet. I marked them as to skip and we can look at them later. I'd rather get in the baseline tests sooner, rather than spending a bunch of time looking at that. The output differences also seem very minor, so not super concerning. The biggest remaining issue is that this doesn't interact well with the Chdir we're doing in the engine. We could either pass the CWD through, or just try to get rid of that Chdir. So this should only be merged after https://github.com/pulumi/pulumi/pull/15607. I've tried to split this into a few commits, separating out adding the testdata, so it's hopefully a little easier to review, even though the PR is still quite large. One other thing to note is that we're comparing that the output has all the same lines, and not that it is exactly the same. Because of how the engine is implemented, there's a bunch of race conditions otherwise, that would make us have to skip a bunch of tests, just because e.g. resource A is sometimes deleted before resource B and sometimes it's the other way around. The biggest downside of that is that running with `PULUMI_ACCEPT` will produce a diff even when there are no changes. Hopefully we won't have to run that way too often though, so it might not be a huge issue? --------- Co-authored-by: Fraser Waters <fraser@pulumi.com>
2024-05-13 07:18:25 +00:00
// Skip display tests because different ordering makes the colouring different.
Options: TestUpdateOptions{T: t, HostF: hostF, SkipDisplayTests: true},
}
resURN := p.NewURN("pkgA:m:typA", "resA", "")
// Create an old snapshot with two copies of a resource that share a URN: one that is pending deletion and one
// that is not.
old := &deploy.Snapshot{
Resources: []*resource.State{
{
Type: resURN.Type(),
URN: resURN,
Custom: true,
ID: "1",
Inputs: resource.PropertyMap{},
Outputs: resource.PropertyMap{},
},
{
Type: resURN.Type(),
URN: resURN,
Custom: true,
ID: "0",
Inputs: resource.PropertyMap{},
Outputs: resource.PropertyMap{},
Delete: true,
},
},
}
p.Steps = []TestStep{{
Op: Update,
Validate: func(_ workspace.Project, _ deploy.Target, entries JournalEntries,
_ []Event, err error,
) error {
// Verify that we see a DeleteReplacement for the resource with ID 0 and a Delete for the resource with
// ID 1.
deletedID0, deletedID1 := false, false
for _, entry := range entries {
// Ignore non-terminal steps and steps that affect the injected default provider.
if entry.Kind != JournalEntrySuccess || entry.Step.URN() != resURN ||
(entry.Step.Op() != deploy.OpDelete && entry.Step.Op() != deploy.OpDeleteReplaced) {
continue
}
switch id := entry.Step.Old().ID; id {
case "0":
assert.False(t, deletedID0)
deletedID0 = true
case "1":
assert.False(t, deletedID1)
deletedID1 = true
default:
assert.Fail(t, "unexpected resource ID %v", string(id))
}
}
assert.True(t, deletedID0)
assert.True(t, deletedID1)
return err
},
}}
p.Run(t, old)
}
func TestUpdateWithPendingDelete(t *testing.T) {
t.Parallel()
loaders := []*deploytest.ProviderLoader{
deploytest.NewProviderLoader("pkgA", semver.MustParse("1.0.0"), func() (plugin.Provider, error) {
return &deploytest.Provider{}, nil
}),
}
Lifecycle tests shouldn't use a closed host (#14063) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This PR fixes the inadvertent use of a closed plugin host in the lifecycle tests. The tests override the host that is provided to the engine, for good reasons, but that same host is re-used across multiple engine operations. Since the engine closes the supplied host at the end of each operation, subsequent operations are handed a closed host. In order to detect engine bugs related to the use of a closed host (see https://github.com/pulumi/pulumi/pull/14057), the fake host should return an error if it is used after being closed (as does the real host). This PR addresses this. The detailed change is to shift to using a host factory that produces a host in `TestOp.Run`. The `TestPlan` now takes a `TestUpdateOptions` with `HostF` and an embedded `UpdateOptions`. Note that two tests fail due to https://github.com/pulumi/pulumi/pull/14057 which was being masked by the problem that is fixed here. This PR disables those tests and the other PR will re-enable them. - `TestCanceledRefresh` - `TestProviderCancellation` ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-28 21:50:18 +00:00
hostF := deploytest.NewPluginHostF(nil, nil, nil, loaders...)
p := &TestPlan{
Add display to the engine tests (#16050) We want to add more test coverage to the display code. The best way to do that is to add it to the engine tests, that already cover most of the pulumi functionality. It's probably not really possible to review all of the output, but at least it gives us a baseline, which we can work with. There's a couple of tests that are flaky for reasons I don't quite understand yet. I marked them as to skip and we can look at them later. I'd rather get in the baseline tests sooner, rather than spending a bunch of time looking at that. The output differences also seem very minor, so not super concerning. The biggest remaining issue is that this doesn't interact well with the Chdir we're doing in the engine. We could either pass the CWD through, or just try to get rid of that Chdir. So this should only be merged after https://github.com/pulumi/pulumi/pull/15607. I've tried to split this into a few commits, separating out adding the testdata, so it's hopefully a little easier to review, even though the PR is still quite large. One other thing to note is that we're comparing that the output has all the same lines, and not that it is exactly the same. Because of how the engine is implemented, there's a bunch of race conditions otherwise, that would make us have to skip a bunch of tests, just because e.g. resource A is sometimes deleted before resource B and sometimes it's the other way around. The biggest downside of that is that running with `PULUMI_ACCEPT` will produce a diff even when there are no changes. Hopefully we won't have to run that way too often though, so it might not be a huge issue? --------- Co-authored-by: Fraser Waters <fraser@pulumi.com>
2024-05-13 07:18:25 +00:00
// Skip display tests because different ordering makes the colouring different.
Options: TestUpdateOptions{T: t, HostF: hostF, SkipDisplayTests: true},
}
resURN := p.NewURN("pkgA:m:typA", "resA", "")
// Create an old snapshot with two copies of a resource that share a URN: one that is pending deletion and one
// that is not.
old := &deploy.Snapshot{
Resources: []*resource.State{
{
Type: resURN.Type(),
URN: resURN,
Custom: true,
ID: "1",
Inputs: resource.PropertyMap{},
Outputs: resource.PropertyMap{},
},
{
Type: resURN.Type(),
URN: resURN,
Custom: true,
ID: "0",
Inputs: resource.PropertyMap{},
Outputs: resource.PropertyMap{},
Delete: true,
},
},
}
p.Steps = []TestStep{{
Op: Destroy,
Validate: func(_ workspace.Project, _ deploy.Target, entries JournalEntries,
_ []Event, err error,
) error {
// Verify that we see a DeleteReplacement for the resource with ID 0 and a Delete for the resource with
// ID 1.
deletedID0, deletedID1 := false, false
for _, entry := range entries {
// Ignore non-terminal steps and steps that affect the injected default provider.
if entry.Kind != JournalEntrySuccess || entry.Step.URN() != resURN ||
(entry.Step.Op() != deploy.OpDelete && entry.Step.Op() != deploy.OpDeleteReplaced) {
continue
}
switch id := entry.Step.Old().ID; id {
case "0":
assert.False(t, deletedID0)
deletedID0 = true
case "1":
assert.False(t, deletedID1)
deletedID1 = true
default:
assert.Fail(t, "unexpected resource ID %v", string(id))
}
}
assert.True(t, deletedID0)
assert.True(t, deletedID1)
return err
},
}}
p.Run(t, old)
}