mirrors
/
pulumi
mirror of https://github.com/pulumi/pulumi.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
pulumi/tests/integration/integration_test.go

834 lines
24 KiB
Go
Raw Permalink Blame History

// Copyright 2016-2021, Pulumi Corporation. All rights reserved.
package ints
import (
"bufio"
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
"github.com/pulumi/pulumi/pkg/v3/testing/integration"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
ptesting "github.com/pulumi/pulumi/sdk/v3/go/common/testing"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
)
const WindowsOS = "windows"
// assertPerfBenchmark implements the integration.TestStatsReporter interface, and reports test
// failures when a scenario exceeds the provided threshold.
type assertPerfBenchmark struct {
T *testing.T
MaxPreviewDuration time.Duration
MaxUpdateDuration time.Duration
}
func (t assertPerfBenchmark) ReportCommand(stats integration.TestCommandStats) {
var maxDuration *time.Duration
if strings.HasPrefix(stats.StepName, "pulumi-preview") {
maxDuration = &t.MaxPreviewDuration
}
if strings.HasPrefix(stats.StepName, "pulumi-update") {
maxDuration = &t.MaxUpdateDuration
}
if maxDuration != nil && *maxDuration != 0 {
if stats.ElapsedSeconds < maxDuration.Seconds() {
t.T.Logf(
"Test step %q was under threshold. %.2fs (max %.2fs)",
stats.StepName, stats.ElapsedSeconds, maxDuration.Seconds())
} else {
t.T.Errorf(
"Test step %q took longer than expected. %.2fs vs. max %.2fs",
stats.StepName, stats.ElapsedSeconds, maxDuration.Seconds())
}
}
}
// TestStackTagValidation verifies various error scenarios related to stack names and tags.
func TestStackTagValidation(t *testing.T) {
t.Run("Error_StackName", func(t *testing.T) {
e := ptesting.NewEnvironment(t)
defer func() {
if !t.Failed() {
e.DeleteEnvironment()
}
}()
e.RunCommand("git", "init")
e.ImportDirectory("stack_project_name")
e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL())
stdout, stderr := e.RunCommandExpectError("pulumi", "stack", "init", "invalid name (spaces, parens, etc.)")
assert.Equal(t, "", stdout)
assert.Contains(t, stderr, "stack names may only contain alphanumeric, hyphens, underscores, or periods")
})
t.Run("Error_DescriptionLength", func(t *testing.T) {
e := ptesting.NewEnvironment(t)
defer func() {
if !t.Failed() {
e.DeleteEnvironment()
}
}()
e.RunCommand("git", "init")
e.ImportDirectory("stack_project_name")
e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL())
prefix := "lorem ipsum dolor sit amet" // 26
prefix = prefix + prefix + prefix + prefix // 104
prefix = prefix + prefix + prefix + prefix // 416 + the current Pulumi.yaml's description
// Change the contents of the Description property of Pulumi.yaml.
yamlPath := filepath.Join(e.CWD, "Pulumi.yaml")
err := integration.ReplaceInFile("description: ", "description: "+prefix, yamlPath)
assert.NoError(t, err)
stdout, stderr := e.RunCommandExpectError("pulumi", "stack", "init", "valid-name")
assert.Equal(t, "", stdout)
assert.Contains(t, stderr, "error: could not create stack:")
assert.Contains(t, stderr, "validating stack properties:")
assert.Contains(t, stderr, "stack tag \"pulumi:description\" value is too long (max length 256 characters)")
})
}
// TestConfigSave ensures that config commands in the Pulumi CLI work as expected.
func TestConfigSave(t *testing.T) {
e := ptesting.NewEnvironment(t)
defer func() {
if !t.Failed() {
e.DeleteEnvironment()
}
}()
// Initialize an empty stack.
path := filepath.Join(e.RootPath, "Pulumi.yaml")
err := (&workspace.Project{
Name: "testing-config",
Runtime: workspace.NewProjectRuntimeInfo("nodejs", nil),
}).Save(path)
assert.NoError(t, err)
e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL())
e.RunCommand("pulumi", "stack", "init", "testing-2")
e.RunCommand("pulumi", "stack", "init", "testing-1")
// Now configure and save a few different things:
e.RunCommand("pulumi", "config", "set", "configA", "value1")
e.RunCommand("pulumi", "config", "set", "configB", "value2", "--stack", "testing-2")
e.RunCommand("pulumi", "stack", "select", "testing-2")
e.RunCommand("pulumi", "config", "set", "configD", "value4")
e.RunCommand("pulumi", "config", "set", "configC", "value3", "--stack", "testing-1")
// Now read back the config using the CLI:
{
stdout, _ := e.RunCommand("pulumi", "config", "get", "configB")
assert.Equal(t, "value2\n", stdout)
}
{
// the config in a different stack, so this should error.
stdout, stderr := e.RunCommandExpectError("pulumi", "config", "get", "configA")
assert.Equal(t, "", stdout)
assert.NotEqual(t, "", stderr)
}
{
// but selecting the stack should let you see it
stdout, _ := e.RunCommand("pulumi", "config", "get", "configA", "--stack", "testing-1")
assert.Equal(t, "value1\n", stdout)
}
// Finally, check that the stack file contains what we expected.
validate := func(k string, v string, cfg config.Map) {
key, err := config.ParseKey("testing-config:config:" + k)
assert.NoError(t, err)
d, ok := cfg[key]
assert.True(t, ok, "config key %v should be set", k)
dv, err := d.Value(nil)
assert.NoError(t, err)
assert.Equal(t, v, dv)
}
testStack1, err := workspace.LoadProjectStack(filepath.Join(e.CWD, "Pulumi.testing-1.yaml"))
assert.NoError(t, err)
testStack2, err := workspace.LoadProjectStack(filepath.Join(e.CWD, "Pulumi.testing-2.yaml"))
assert.NoError(t, err)
assert.Equal(t, 2, len(testStack1.Config))
assert.Equal(t, 2, len(testStack2.Config))
validate("configA", "value1", testStack1.Config)
validate("configC", "value3", testStack1.Config)
validate("configB", "value2", testStack2.Config)
validate("configD", "value4", testStack2.Config)
e.RunCommand("pulumi", "stack", "rm", "--yes")
}
// TestConfigPaths ensures that config commands with paths work as expected.
func TestConfigPaths(t *testing.T) {
e := ptesting.NewEnvironment(t)
defer func() {
if !t.Failed() {
e.DeleteEnvironment()
}
}()
// Initialize an empty stack.
path := filepath.Join(e.RootPath, "Pulumi.yaml")
err := (&workspace.Project{
Name: "testing-config",
Runtime: workspace.NewProjectRuntimeInfo("nodejs", nil),
}).Save(path)
assert.NoError(t, err)
e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL())
e.RunCommand("pulumi", "stack", "init", "testing")
namespaces := []string{"", "my:"}
tests := []struct {
Key string
Value string
Secret bool
Path bool
TopLevelKey string
TopLevelExpectedValue string
}{
{
Key: "aConfigValue",
Value: "this value is a value",
TopLevelKey: "aConfigValue",
TopLevelExpectedValue: "this value is a value",
},
{
Key: "anotherConfigValue",
Value: "this value is another value",
TopLevelKey: "anotherConfigValue",
TopLevelExpectedValue: "this value is another value",
},
{
Key: "bEncryptedSecret",
Value: "this super secret is encrypted",
Secret: true,
TopLevelKey: "bEncryptedSecret",
TopLevelExpectedValue: "this super secret is encrypted",
},
{
Key: "anotherEncryptedSecret",
Value: "another encrypted secret",
Secret: true,
TopLevelKey: "anotherEncryptedSecret",
TopLevelExpectedValue: "another encrypted secret",
},
{
Key: "[]",
Value: "square brackets value",
TopLevelKey: "[]",
TopLevelExpectedValue: "square brackets value",
},
{
Key: "x.y",
Value: "x.y value",
TopLevelKey: "x.y",
TopLevelExpectedValue: "x.y value",
},
{
Key: "0",
Value: "0 value",
Path: true,
TopLevelKey: "0",
TopLevelExpectedValue: "0 value",
},
{
Key: "true",
Value: "value",
Path: true,
TopLevelKey: "true",
TopLevelExpectedValue: "value",
},
{
Key: `["test.Key"]`,
Value: "test key value",
Path: true,
TopLevelKey: "test.Key",
TopLevelExpectedValue: "test key value",
},
{
Key: `nested["test.Key"]`,
Value: "nested test key value",
Path: true,
TopLevelKey: "nested",
TopLevelExpectedValue: `{"test.Key":"nested test key value"}`,
},
{
Key: "outer.inner",
Value: "value",
Path: true,
TopLevelKey: "outer",
TopLevelExpectedValue: `{"inner":"value"}`,
},
{
Key: "names[0]",
Value: "a",
Path: true,
TopLevelKey: "names",
TopLevelExpectedValue: `["a"]`,
},
{
Key: "names[1]",
Value: "b",
Path: true,
TopLevelKey: "names",
TopLevelExpectedValue: `["a","b"]`,
},
{
Key: "names[2]",
Value: "c",
Path: true,
TopLevelKey: "names",
TopLevelExpectedValue: `["a","b","c"]`,
},
{
Key: "names[3]",
Value: "super secret name",
Path: true,
Secret: true,
TopLevelKey: "names",
TopLevelExpectedValue: `["a","b","c","super secret name"]`,
},
{
Key: "servers[0].port",
Value: "80",
Path: true,
TopLevelKey: "servers",
TopLevelExpectedValue: `[{"port":80}]`,
},
{
Key: "servers[0].host",
Value: "example",
Path: true,
TopLevelKey: "servers",
TopLevelExpectedValue: `[{"host":"example","port":80}]`,
},
{
Key: "a.b[0].c",
Value: "true",
Path: true,
TopLevelKey: "a",
TopLevelExpectedValue: `{"b":[{"c":true}]}`,
},
{
Key: "a.b[1].c",
Value: "false",
Path: true,
TopLevelKey: "a",
TopLevelExpectedValue: `{"b":[{"c":true},{"c":false}]}`,
},
{
Key: "tokens[0]",
Value: "shh",
Path: true,
Secret: true,
TopLevelKey: "tokens",
TopLevelExpectedValue: `["shh"]`,
},
{
Key: "foo.bar",
Value: "don't tell",
Path: true,
Secret: true,
TopLevelKey: "foo",
TopLevelExpectedValue: `{"bar":"don't tell"}`,
},
{
Key: "semiInner.a.b.c.d",
Value: "1",
Path: true,
TopLevelKey: "semiInner",
TopLevelExpectedValue: `{"a":{"b":{"c":{"d":1}}}}`,
},
{
Key: "wayInner.a.b.c.d.e.f.g.h.i.j.k",
Value: "false",
Path: true,
TopLevelKey: "wayInner",
TopLevelExpectedValue: `{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"h":{"i":{"j":{"k":false}}}}}}}}}}}`,
},
{
Key: "foo1[0]",
Value: "false",
Path: true,
TopLevelKey: "foo1",
TopLevelExpectedValue: `[false]`,
},
{
Key: "foo2[0]",
Value: "true",
Path: true,
TopLevelKey: "foo2",
TopLevelExpectedValue: `[true]`,
},
{
Key: "foo3[0]",
Value: "10",
Path: true,
TopLevelKey: "foo3",
TopLevelExpectedValue: `[10]`,
},
{
Key: "foo4[0]",
Value: "0",
Path: true,
TopLevelKey: "foo4",
TopLevelExpectedValue: `[0]`,
},
{
Key: "foo5[0]",
Value: "00",
Path: true,
TopLevelKey: "foo5",
TopLevelExpectedValue: `["00"]`,
},
{
Key: "foo6[0]",
Value: "01",
Path: true,
TopLevelKey: "foo6",
TopLevelExpectedValue: `["01"]`,
},
{
Key: "foo7[0]",
Value: "0123456",
Path: true,
TopLevelKey: "foo7",
TopLevelExpectedValue: `["0123456"]`,
},
{
Key: "bar1.inner",
Value: "false",
Path: true,
TopLevelKey: "bar1",
TopLevelExpectedValue: `{"inner":false}`,
},
{
Key: "bar2.inner",
Value: "true",
Path: true,
TopLevelKey: "bar2",
TopLevelExpectedValue: `{"inner":true}`,
},
{
Key: "bar3.inner",
Value: "10",
Path: true,
TopLevelKey: "bar3",
TopLevelExpectedValue: `{"inner":10}`,
},
{
Key: "bar4.inner",
Value: "0",
Path: true,
TopLevelKey: "bar4",
TopLevelExpectedValue: `{"inner":0}`,
},
{
Key: "bar5.inner",
Value: "00",
Path: true,
TopLevelKey: "bar5",
TopLevelExpectedValue: `{"inner":"00"}`,
},
{
Key: "bar6.inner",
Value: "01",
Path: true,
TopLevelKey: "bar6",
TopLevelExpectedValue: `{"inner":"01"}`,
},
{
Key: "bar7.inner",
Value: "0123456",
Path: true,
TopLevelKey: "bar7",
TopLevelExpectedValue: `{"inner":"0123456"}`,
},
// Overwriting a top-level string value is allowed.
{
Key: "aConfigValue.inner",
Value: "new value",
Path: true,
TopLevelKey: "aConfigValue",
TopLevelExpectedValue: `{"inner":"new value"}`,
},
{
Key: "anotherConfigValue[0]",
Value: "new value",
Path: true,
TopLevelKey: "anotherConfigValue",
TopLevelExpectedValue: `["new value"]`,
},
{
Key: "bEncryptedSecret.inner",
Value: "new value",
Path: true,
TopLevelKey: "bEncryptedSecret",
TopLevelExpectedValue: `{"inner":"new value"}`,
},
{
Key: "anotherEncryptedSecret[0]",
Value: "new value",
Path: true,
TopLevelKey: "anotherEncryptedSecret",
TopLevelExpectedValue: `["new value"]`,
},
}
validateConfigGet := func(key string, value string, path bool) {
args := []string{"config", "get", key}
if path {
args = append(args, "--path")
}
stdout, stderr := e.RunCommand("pulumi", args...)
assert.Equal(t, fmt.Sprintf("%s\n", value), stdout)
assert.Equal(t, "", stderr)
}
for _, ns := range namespaces {
for _, test := range tests {
key := fmt.Sprintf("%s%s", ns, test.Key)
topLevelKey := fmt.Sprintf("%s%s", ns, test.TopLevelKey)
// Set the value.
args := []string{"config", "set"}
if test.Secret {
args = append(args, "--secret")
}
if test.Path {
args = append(args, "--path")
}
args = append(args, key, test.Value)
stdout, stderr := e.RunCommand("pulumi", args...)
assert.Equal(t, "", stdout)
assert.Equal(t, "", stderr)
// Get the value and validate it.
validateConfigGet(key, test.Value, test.Path)
// Get the top-level value and validate it.
validateConfigGet(topLevelKey, test.TopLevelExpectedValue, false /*path*/)
}
}
badKeys := []string{
// Syntax errors.
"root[",
`root["nested]`,
"root.array[abc]",
"root.[1]",
// First path segment must be a non-empty string.
`[""]`,
"[0]",
// Index out of range.
"names[-1]",
"names[5]",
// A "secure" key that is a map with a single string value is reserved by the system.
"key.secure",
"super.nested.map.secure",
// Type mismatch.
"outer[0]",
"names.nested",
"outer.inner.nested",
"outer.inner[0]",
}
for _, ns := range namespaces {
for _, badKey := range badKeys {
key := fmt.Sprintf("%s%s", ns, badKey)
stdout, stderr := e.RunCommandExpectError("pulumi", "config", "set", "--path", key, "value")
assert.Equal(t, "", stdout)
assert.NotEqual(t, "", stderr)
}
}
e.RunCommand("pulumi", "stack", "rm", "--yes")
}
//nolint:deadcode
func pathEnv(t *testing.T, path ...string) string {
pathEnv := []string{os.Getenv("PATH")}
for _, p := range path {
absPath, err := filepath.Abs(p)
if err != nil {
t.Fatal(err)
return ""
}
pathEnv = append(pathEnv, absPath)
}
pathSeparator := ":"
if runtime.GOOS == "windows" {
pathSeparator = ";"
}
return "PATH=" + strings.Join(pathEnv, pathSeparator)
}
//nolint:deadcode
func testComponentSlowPathEnv(t *testing.T) string {
return pathEnv(t, filepath.Join("construct_component_slow", "testcomponent"))
}
//nolint:deadcode
func testComponentPlainPathEnv(t *testing.T) string {
return pathEnv(t, filepath.Join("construct_component_plain", "testcomponent"))
}
// nolint: unused,deadcode
func venvFromPipenv(relativeWorkdir string) (string, error) {
workdir, err := filepath.Abs(relativeWorkdir)
if err != nil {
return "", err
}
cmd := exec.Command("pipenv", "--venv")
cmd.Dir = workdir
dir, err := cmd.Output()
if err != nil {
return "", err
}
venv := strings.TrimRight(string(dir), "\r\n")
if _, err := os.Stat(venv); os.IsNotExist(err) {
return "", fmt.Errorf("Folder '%s' returned by 'pipenv --venv' from %s does not exist: %w",
venv, workdir, err)
}
return venv, nil
}
// nolint: unused,deadcode
func pulumiRuntimeVirtualEnv(t *testing.T, pulumiRepoRootDir string) string {
venvFolder, err := venvFromPipenv(filepath.Join(pulumiRepoRootDir, "sdk", "python"))
if err != nil {
t.Fatal(fmt.Errorf("PULUMI_RUNTIME_VIRTUALENV guess failed: %w", err))
return ""
}
r := fmt.Sprintf("PULUMI_RUNTIME_VIRTUALENV=%s", venvFolder)
return r
}
// nolint: unused,deadcode
func testComponentProviderSchema(t *testing.T, path string, env ...string) {
tests := []struct {
name string
env []string
version int32
expected string
expectedError string
}{
{
name: "Default",
expected: "{}",
},
{
name: "Schema",
env: []string{"INCLUDE_SCHEMA=true"},
expected: `{"hello": "world"}`,
},
{
name: "Invalid Version",
version: 15,
expectedError: "unsupported schema version 15",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
// Start the plugin binary.
cmd := exec.Command(path, "ignored")
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, env...)
cmd.Env = append(cmd.Env, test.env...)
stdout, err := cmd.StdoutPipe()
assert.NoError(t, err)
err = cmd.Start()
assert.NoError(t, err)
defer func() {
// Ignore the error as it may fail with access denied on Windows.
cmd.Process.Kill() // nolint: errcheck
}()
// Read the port from standard output.
reader := bufio.NewReader(stdout)
bytes, err := reader.ReadBytes('\n')
assert.NoError(t, err)
port := strings.TrimSpace(string(bytes))
// Create a connection to the server.
conn, err := grpc.Dial("127.0.0.1:"+port, grpc.WithInsecure(), rpcutil.GrpcChannelOptions())
assert.NoError(t, err)
client := pulumirpc.NewResourceProviderClient(conn)
// Call GetSchema and verify the results.
resp, err := client.GetSchema(context.Background(), &pulumirpc.GetSchemaRequest{Version: test.version})
if test.expectedError != "" {
assert.Error(t, err)
assert.Contains(t, err.Error(), test.expectedError)
} else {
assert.Equal(t, test.expected, resp.GetSchema())
}
})
}
}
// Test remote component inputs properly handle unknowns.
// nolint: unused,deadcode
func testConstructUnknown(t *testing.T, lang string, dependencies ...string) {
const testDir = "construct_component_unknown"
tests := []struct {
componentDir string
env []string
}{
{
componentDir: "testcomponent",
},
{
componentDir: "testcomponent-python",
env: []string{pulumiRuntimeVirtualEnv(t, filepath.Join("..", ".."))},
},
{
componentDir: "testcomponent-go",
},
}
for _, test := range tests {
t.Run(test.componentDir, func(t *testing.T) {
pathEnv := pathEnv(t,
filepath.Join("..", "testprovider"),
filepath.Join(testDir, test.componentDir))
integration.ProgramTest(t, &integration.ProgramTestOptions{
Env: append(test.env, pathEnv),
Dir: filepath.Join(testDir, lang),
Dependencies: dependencies,
SkipRefresh: true,
SkipPreview: false,
SkipUpdate: true,
SkipExportImport: true,
SkipEmptyPreviewUpdate: true,
Quick: false,
NoParallel: true,
})
})
}
}
// Test methods properly handle unknowns.
// nolint: unused,deadcode
func testConstructMethodsUnknown(t *testing.T, lang string, dependencies ...string) {
const testDir = "construct_component_methods_unknown"
tests := []struct {
componentDir string
env []string
}{
{
componentDir: "testcomponent",
},
{
componentDir: "testcomponent-python",
env: []string{pulumiRuntimeVirtualEnv(t, filepath.Join("..", ".."))},
},
{
componentDir: "testcomponent-go",
},
}
for _, test := range tests {
t.Run(test.componentDir, func(t *testing.T) {
pathEnv := pathEnv(t,
filepath.Join("..", "testprovider"),
filepath.Join(testDir, test.componentDir))
integration.ProgramTest(t, &integration.ProgramTestOptions{
Env: append(test.env, pathEnv),
Dir: filepath.Join(testDir, lang),
Dependencies: dependencies,
SkipRefresh: true,
SkipPreview: false,
SkipUpdate: true,
SkipExportImport: true,
SkipEmptyPreviewUpdate: true,
Quick: false,
NoParallel: true,
})
})
}
}
func TestRotatePassphrase(t *testing.T) {
e := ptesting.NewEnvironment(t)
defer func() {
if !t.Failed() {
e.DeleteEnvironment()
}
}()
e.ImportDirectory("rotate_passphrase")
e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL())
e.RunCommand("pulumi", "stack", "init", "dev")
e.RunCommand("pulumi", "up", "--skip-preview", "--yes")
e.RunCommand("pulumi", "config", "set", "--secret", "foo", "bar")
e.SetEnvVars([]string{"PULUMI_TEST_PASSPHRASE=true"})
e.Stdin = strings.NewReader("qwerty\nqwerty\n")
e.RunCommand("pulumi", "stack", "change-secrets-provider", "passphrase")
e.Stdin, e.Passphrase = nil, "qwerty"
e.RunCommand("pulumi", "config", "get", "foo")
}
func TestReplaceOnChanges(t *testing.T) {
for _, tt := range []struct {
lang string
main string
}{
{"dotnet", "MyStack.cs"},
{"python", "__main__.py"},
{"go", "main.go"},
{"nodejs", "index.ts"},
} {
t.Run(tt.lang, func(t *testing.T) {
e := ptesting.NewEnvironment(t)
defer func() {
if !t.Failed() {
e.DeleteEnvironment()
}
}()
e.ImportDirectory(filepath.Join("replace_on_changes", tt.lang))
e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL())
e.RunCommand("pulumi", "stack", "init", "dev")
e.RunCommand("pulumi", "config", "set", "aws:region", "us-west-2")
if tt.lang == "nodejs" {
e.RunCommand("yarn", "install")
}
e.RunCommand("pulumi", "up", "--skip-preview", "--non-interactive", "--yes")
err := os.Rename(tt.main+".changed", tt.main)
assert.NoError(t, err)
stdout, _ := e.RunCommand("pulumi", "preview", "--non-interactive", "--color=never")
assert.Contains(t, stdout, "+-1 to replace")
})
}
}
Reference in New Issue View Git Blame Copy Permalink