pulumi/sdk/nodejs/cmd/pulumi-language-nodejs/main.go

1563 lines
53 KiB
Go
Raw Permalink Normal View History

// Copyright 2016-2022, Pulumi Corporation.
2018-05-22 19:43:36 +00:00
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// pulumi-language-nodejs serves as the "language host" for Pulumi
// programs written in NodeJS. It is ultimately responsible for spawning the
// language runtime that executes the program.
//
// The program being executed is executed by a shim script called
// `pulumi-language-nodejs-exec`. This script is written in the hosted
// language (in this case, node) and is responsible for initiating RPC
// links to the resource monitor and engine.
//
// It's therefore the responsibility of this program to implement
// the LanguageHostServer endpoint by spawning instances of
// `pulumi-language-nodejs-exec` and forwarding the RPC request arguments
// to the command-line.
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
2022-11-04 10:20:29 +00:00
"errors"
"flag"
"fmt"
"io"
"io/fs"
"os"
"os/exec"
"os/signal"
"path/filepath"
Enable perfsprint linter (#14813) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Prompted by a comment in another review: https://github.com/pulumi/pulumi/pull/14654#discussion_r1419995945 This lints that we don't use `fmt.Errorf` when `errors.New` will suffice, it also covers a load of other cases where `Sprintf` is sub-optimal. Most of these edits were made by running `perfsprint --fix`. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-12 12:19:42 +00:00
"strconv"
"strings"
"time"
"github.com/blang/semver"
"github.com/google/shlex"
"github.com/hashicorp/go-multierror"
opentracing "github.com/opentracing/opentracing-go"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
Remove deprecated Protobufs imports (#15158) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> github.com/golang/protobuf is marked deprecated and I was getting increasingly triggered by the inconsistency of importing the `Empty` type from "github.com/golang/protobuf/ptypes/empty" or "google.golang.org/protobuf/types/known/emptypb" as "pbempty" or "empty" or "emptypb". Similar for the struct type. So this replaces all the Protobufs imports with ones from "google.golang.org/protobuf", normalises the import name to always just be the module name (emptypb), and adds the depguard linter to ensure we don't use the deprecated package anymore. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-01-17 09:35:20 +00:00
"google.golang.org/protobuf/types/known/emptypb"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/executable"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/fsutil"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
"github.com/pulumi/pulumi/sdk/v3/go/common/version"
2023-05-12 13:38:36 +00:00
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
"github.com/pulumi/pulumi/sdk/v3/nodejs/npm"
pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
2023-05-12 13:38:36 +00:00
hclsyntax "github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax"
codegen "github.com/pulumi/pulumi/pkg/v3/codegen/nodejs"
"github.com/pulumi/pulumi/pkg/v3/codegen/pcl"
"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
)
const (
// The path to the "run" program which will spawn the rest of the language host. This may be overridden with
// PULUMI_LANGUAGE_NODEJS_RUN_PATH, which we do in some testing cases.
defaultRunPath = "@pulumi/pulumi/cmd/run"
// The runtime expects the config object to be saved to this environment variable.
pulumiConfigVar = "PULUMI_CONFIG"
// The runtime expects the array of secret config keys to be saved to this environment variable.
sdk/go: Remove 'nolint' directives from package docs Go treats comments that match the following regex as directives. //[a-z0-9]+:[a-z0-9] Comments that are directives don't show in an entity's documentation. https://github.com/golang/go/commit/5a550b695117f07a4f2454039a4871250cd3ed09#diff-f56160fd9fcea272966a8a1d692ad9f49206fdd8dbcbfe384865a98cd9bc2749R165 Our code has `//nolint` directives that now show in the API Reference. This is because these directives are in one of the following forms, which don't get this special treatment. // nolint:foo //nolint: foo This change fixes all such directives found by the regex: `// nolint|//nolint: `. See bottom of commit for command used for the fix. Verification: Here's the output of `go doc` on some entities before and after this change. Before ``` % go doc github.com/pulumi/pulumi/sdk/v3/go/pulumi | head -n8 package pulumi // import "github.com/pulumi/pulumi/sdk/v3/go/pulumi" nolint: lll, interfacer nolint: lll, interfacer const EnvOrganization = "PULUMI_ORGANIZATION" ... var ErrPlugins = errors.New("pulumi: plugins requested") ``` After ``` % go doc github.com/pulumi/pulumi/sdk/v3/go/pulumi | head -n8 package pulumi // import "github.com/pulumi/pulumi/sdk/v3/go/pulumi" const EnvOrganization = "PULUMI_ORGANIZATION" ... var ErrPlugins = errors.New("pulumi: plugins requested") func BoolRef(v bool) *bool func Float64Ref(v float64) *float64 func IntRef(v int) *int func IsSecret(o Output) bool ``` Before ``` % go doc github.com/pulumi/pulumi/sdk/v3/go/pulumi URN_ package pulumi // import "github.com/pulumi/pulumi/sdk/v3/go/pulumi" func URN_(o string) ResourceOption URN_ is an optional URN of a previously-registered resource of this type to read from the engine. nolint: revive ``` After: ``` % go doc github.com/pulumi/pulumi/sdk/v3/go/pulumi URN_ package pulumi // import "github.com/pulumi/pulumi/sdk/v3/go/pulumi" func URN_(o string) ResourceOption URN_ is an optional URN of a previously-registered resource of this type to read from the engine. ``` Note that golangci-lint offers a 'nolintlint' linter that finds such miuses of nolint, but it also finds other issues so I've deferred that to a follow up PR. Resolves #11785 Related: https://github.com/golangci/golangci-lint/issues/892 [git-generate] FILES=$(mktemp) rg -l '// nolint|//nolint: ' | tee "$FILES" | xargs perl -p -i -e ' s|// nolint|//nolint|g; s|//nolint: |//nolint:|g; ' rg '.go$' < "$FILES" | xargs gofmt -w -s
2023-01-06 00:07:45 +00:00
//nolint:gosec
pulumiConfigSecretKeysVar = "PULUMI_CONFIG_SECRET_KEYS"
// A exit-code we recognize when the nodejs process exits. If we see this error, there's no
// need for us to print any additional error messages since the user already got a a good
// one they can handle.
nodeJSProcessExitedAfterShowingUserActionableMessage = 32
)
// Launches the language host RPC endpoint, which in turn fires
// up an RPC server implementing the LanguageRuntimeServer RPC
// endpoint.
func main() {
var tracing string
flag.StringVar(&tracing, "tracing", "",
"Emit tracing to a Zipkin-compatible tracing endpoint")
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
flag.Bool("typescript", true,
"[obsolete] Use ts-node at runtime to support typescript source natively")
flag.String("root", "", "[obsolete] Project root path to use")
flag.String("tsconfig", "",
"[obsolete] Path to tsconfig.json to use")
flag.String("nodeargs", "", "[obsolete] Arguments for the Node process")
flag.String("packagemanager", "", "[obsolete] Packagemanager to use (auto, npm, yarn or pnpm)")
flag.Parse()
args := flag.Args()
logging.InitLogging(false, 0, false)
cmdutil.InitTracing("pulumi-language-nodejs", "pulumi-language-nodejs", tracing)
// Optionally pluck out the engine so we can do logging, etc.
var engineAddress string
if len(args) > 0 {
engineAddress = args[0]
}
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
// map the context Done channel to the rpcutil boolean cancel channel
cancelChannel := make(chan bool)
go func() {
<-ctx.Done()
cancel() // deregister the interrupt handler
close(cancelChannel)
}()
err := rpcutil.Healthcheck(ctx, engineAddress, 5*time.Minute, cancel)
if err != nil {
2022-11-04 10:20:29 +00:00
cmdutil.Exit(fmt.Errorf("could not start health check host RPC server: %w", err))
}
// Fire up a gRPC server, letting the kernel choose a free port.
2022-11-01 15:15:09 +00:00
handle, err := rpcutil.ServeWithOptions(rpcutil.ServeOptions{
Cancel: cancelChannel,
Init: func(srv *grpc.Server) error {
Run conformance tests with and without ts-node (#15809) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/15075. This doubles up the conformance tests to run with `tsc` instead of `ts-node`. Getting this to work in the conformance framework required a bit of pushing the envelope for what it means to "install dependencies" and "run", but it works for testing, and isn't horribly exposed to users. It also points out a couple of things that we might want to clean up for users wanting the same workflow. 1. Adding a `Build` command (which we might want for the type checker work added in https://github.com/pulumi/pulumi/pull/15725 as well). 2. If you set "main" to ./bin because you're using tsc, you can't then use InstallDependencies because to start with the "bin" folder is missing. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-03-28 15:44:25 +00:00
host := newLanguageHost(engineAddress, tracing, false /* forceTsc */)
pulumirpc.RegisterLanguageRuntimeServer(srv, host)
return nil
},
2022-11-01 15:15:09 +00:00
Options: rpcutil.OpenTracingServerInterceptorOptions(nil),
})
if err != nil {
2022-11-04 10:20:29 +00:00
cmdutil.Exit(fmt.Errorf("could not start language host RPC server: %w", err))
}
// Otherwise, print out the port so that the spawner knows how to reach us.
2022-11-01 15:15:09 +00:00
fmt.Printf("%d\n", handle.Port)
// And finally wait for the server to stop serving.
2022-11-01 15:15:09 +00:00
if err := <-handle.Done; err != nil {
2022-11-04 10:20:29 +00:00
cmdutil.Exit(fmt.Errorf("language host RPC stopped serving: %w", err))
}
}
// locateModule resolves a node module name to a file path that can be loaded
func locateModule(ctx context.Context, mod, programDir, nodeBin string) (string, error) {
script := fmt.Sprintf(`try {
Return the underlying error if we can't locate the pulumi SDK (#16160) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description When locating the pulumi SDK fails with an error other than MODULE_NOT_FOUND, return the underlying error instead of our default error message asking to run `pulumi install`. This is helpful when there's a typo in package.json and it can't be parsed. Fixes https://github.com/pulumi/pulumi/issues/4973 ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-05-10 13:34:05 +00:00
console.log(require.resolve('%s'));
} catch (error) {
if (error.code === 'MODULE_NOT_FOUND') {
console.error("It looks like the Pulumi SDK has not been installed. Have you run pulumi install?")
} else {
console.error(error.message);
}
process.exit(1);
}`, mod)
// The Volta package manager installs shim executables that route to the user's chosen nodejs
// version. On Windows this does not properly handle arguments with newlines, so we need to
// ensure that the script is a single line.
// https://github.com/pulumi/pulumi/issues/16393
script = strings.Replace(script, "\n", "", -1)
args := []string{"-e", script}
tracingSpan, _ := opentracing.StartSpanFromContext(ctx,
"locateModule",
opentracing.Tag{Key: "module", Value: mod},
opentracing.Tag{Key: "component", Value: "exec.Command"},
opentracing.Tag{Key: "command", Value: nodeBin},
opentracing.Tag{Key: "args", Value: args})
defer tracingSpan.Finish()
cmd := exec.Command(nodeBin, args...)
cmd.Dir = programDir
out, err := cmd.Output()
if err != nil {
Return the underlying error if we can't locate the pulumi SDK (#16160) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description When locating the pulumi SDK fails with an error other than MODULE_NOT_FOUND, return the underlying error instead of our default error message asking to run `pulumi install`. This is helpful when there's a typo in package.json and it can't be parsed. Fixes https://github.com/pulumi/pulumi/issues/4973 ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-05-10 13:34:05 +00:00
if ee, ok := err.(*exec.ExitError); ok {
return "", errors.New(string(ee.Stderr))
}
return "", err
}
return strings.TrimSpace(string(out)), nil
}
// nodeLanguageHost implements the LanguageRuntimeServer interface
// for use as an API endpoint.
type nodeLanguageHost struct {
pulumirpc.UnimplementedLanguageRuntimeServer
engineAddress string
tracing string
Run conformance tests with and without ts-node (#15809) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/15075. This doubles up the conformance tests to run with `tsc` instead of `ts-node`. Getting this to work in the conformance framework required a bit of pushing the envelope for what it means to "install dependencies" and "run", but it works for testing, and isn't horribly exposed to users. It also points out a couple of things that we might want to clean up for users wanting the same workflow. 1. Adding a `Build` command (which we might want for the type checker work added in https://github.com/pulumi/pulumi/pull/15725 as well). 2. If you set "main" to ./bin because you're using tsc, you can't then use InstallDependencies because to start with the "bin" folder is missing. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-03-28 15:44:25 +00:00
// used by language conformance tests to force TSC usage
forceTsc bool
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
}
type nodeOptions struct {
// Use ts-node at runtime to support typescript source natively
typescript bool
// Path to tsconfig.json to use
tsconfigpath string
// Arguments for the Node process
nodeargs string
// The packagemanger to use to install dependencies.
// One of auto, npm, yarn or pnpm, defaults to auto.
packagemanager npm.PackageManagerType
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
}
func parseOptions(options map[string]interface{}) (nodeOptions, error) {
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
// typescript defaults to true
nodeOptions := nodeOptions{
typescript: true,
}
if typescript, ok := options["typescript"]; ok {
if ts, ok := typescript.(bool); ok {
nodeOptions.typescript = ts
} else {
return nodeOptions, errors.New("typescript option must be a boolean")
}
}
if tsconfigpath, ok := options["tsconfig"]; ok {
if tsconfig, ok := tsconfigpath.(string); ok {
nodeOptions.tsconfigpath = tsconfig
} else {
return nodeOptions, errors.New("tsconfigpath option must be a string")
}
}
if nodeargs, ok := options["nodeargs"]; ok {
if args, ok := nodeargs.(string); ok {
nodeOptions.nodeargs = args
} else {
return nodeOptions, errors.New("nodeargs option must be a string")
}
}
if packagemanager, ok := options["packagemanager"]; ok {
if pm, ok := packagemanager.(string); ok {
switch pm {
case "auto":
nodeOptions.packagemanager = npm.AutoPackageManager
case "npm":
nodeOptions.packagemanager = npm.NpmPackageManager
case "yarn":
nodeOptions.packagemanager = npm.YarnPackageManager
case "pnpm":
nodeOptions.packagemanager = npm.PnpmPackageManager
default:
return nodeOptions, fmt.Errorf("packagemanager option must be one of auto, npm, yarn or pnpm, got %q", pm)
}
} else {
return nodeOptions, errors.New("packagemanager option must be a string")
}
} else {
nodeOptions.packagemanager = npm.AutoPackageManager
}
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
return nodeOptions, nil
}
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
func newLanguageHost(
Run conformance tests with and without ts-node (#15809) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/15075. This doubles up the conformance tests to run with `tsc` instead of `ts-node`. Getting this to work in the conformance framework required a bit of pushing the envelope for what it means to "install dependencies" and "run", but it works for testing, and isn't horribly exposed to users. It also points out a couple of things that we might want to clean up for users wanting the same workflow. 1. Adding a `Build` command (which we might want for the type checker work added in https://github.com/pulumi/pulumi/pull/15725 as well). 2. If you set "main" to ./bin because you're using tsc, you can't then use InstallDependencies because to start with the "bin" folder is missing. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-03-28 15:44:25 +00:00
engineAddress, tracing string, forceTsc bool,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
) pulumirpc.LanguageRuntimeServer {
return &nodeLanguageHost{
engineAddress: engineAddress,
tracing: tracing,
Run conformance tests with and without ts-node (#15809) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/15075. This doubles up the conformance tests to run with `tsc` instead of `ts-node`. Getting this to work in the conformance framework required a bit of pushing the envelope for what it means to "install dependencies" and "run", but it works for testing, and isn't horribly exposed to users. It also points out a couple of things that we might want to clean up for users wanting the same workflow. 1. Adding a `Build` command (which we might want for the type checker work added in https://github.com/pulumi/pulumi/pull/15725 as well). 2. If you set "main" to ./bin because you're using tsc, you can't then use InstallDependencies because to start with the "bin" folder is missing. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-03-28 15:44:25 +00:00
forceTsc: forceTsc,
}
}
func compatibleVersions(a, b semver.Version) (bool, string) {
switch {
case a.Major == 0 && b.Major == 0:
// If both major versions are pre-1.0, we require that the major and minor versions match.
if a.Minor != b.Minor {
return false, "Differing major or minor versions are not supported."
}
case a.Major >= 1 && a.Major <= 2 && b.Major >= 1 && b.Major <= 2:
// If both versions are 1.0<=v<=2.0, they are compatible.
case a.Major > 2 || b.Major > 2:
// If either version is post-2.0, we require that the major versions match.
if a.Major != b.Major {
return false, "Differing major versions are not supported."
}
case a.Major == 1 && b.Major == 0 && b.Minor == 17 || b.Major == 1 && a.Major == 0 && a.Minor == 17:
// If one version is pre-1.0 and the other is post-1.0, we unify 1.x.y and 0.17.z. This combination is legal.
default:
// All other combinations of versions are illegal.
return false, "Differing major or minor versions are not supported."
}
return true, ""
}
// GetRequiredPlugins computes the complete set of anticipated plugins required by a program.
func (host *nodeLanguageHost) GetRequiredPlugins(ctx context.Context,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
req *pulumirpc.GetRequiredPluginsRequest,
) (*pulumirpc.GetRequiredPluginsResponse, error) {
// To get the plugins required by a program, find all node_modules/ packages that contain {
// "pulumi": true } inside of their package.json files. We begin this search in the same
// directory that contains the project. It's possible that a developer would do a
// `require("../../elsewhere")` and that we'd miss this as a dependency, however the solution
// for that is simple: install the package in the project root.
// Keep track of the versions of @pulumi/pulumi that are pulled in. If they differ on
// minor version, we will issue a warning to the user.
pulumiPackagePathToVersionMap := make(map[string]semver.Version)
2023-01-23 20:39:24 +00:00
plugins, err := getPluginsFromDir(
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
req.Info.ProgramDirectory,
2023-01-23 20:39:24 +00:00
pulumiPackagePathToVersionMap,
false, /*inNodeModules*/
make(map[string]struct{}))
if err == nil {
first := true
var firstPath string
var firstVersion semver.Version
for path, version := range pulumiPackagePathToVersionMap {
if first {
first = false
firstPath = path
firstVersion = version
continue
}
if ok, message := compatibleVersions(version, firstVersion); !ok {
fmt.Fprintf(os.Stderr,
`Found incompatible versions of @pulumi/pulumi. %s
Version %s referenced at %s
Version %s referenced at %s
`, message, firstVersion, firstPath, version, path)
break
}
}
}
if err != nil {
logging.V(3).Infof("one or more errors while discovering plugins: %s", err)
}
return &pulumirpc.GetRequiredPluginsResponse{
Plugins: plugins,
}, nil
}
// getPluginsFromDir enumerates all node_modules/ directories, deeply, and returns the fully concatenated results.
func getPluginsFromDir(
dir string, pulumiPackagePathToVersionMap map[string]semver.Version,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
inNodeModules bool, visitedPaths map[string]struct{},
) ([]*pulumirpc.PluginDependency, error) {
// try to absolute the input path so visitedPaths can track it correctly
dir, err := filepath.Abs(dir)
if err != nil {
return nil, fmt.Errorf("getting full path for plugin dir %s: %w", dir, err)
}
if _, has := visitedPaths[dir]; has {
return nil, nil
}
visitedPaths[dir] = struct{}{}
files, err := os.ReadDir(dir)
if err != nil {
2022-11-04 10:20:29 +00:00
return nil, fmt.Errorf("reading plugin dir %s: %w", dir, err)
}
var plugins []*pulumirpc.PluginDependency
Do not make errors during plugin discovery fatal The plugin host can ask the language host to provide a list of resource plugins that it thinks will be nessecary for use at deployment time, so they can be eagerly loaded. In NodeJS (the only language host that implements this RPC) This works by walking the directory tree rooted at the CWD of the project, looking for package.json files, parsing them and seeing it they have some marker property set. If they do, we add information about them which we return at the end of our walk. If there is *any* error, the entire operation fails. We've seen a bunch of cases where this happens: - Broken symlinks written by some editors as part of autosave. - Access denied errors when part of the tree is unwalkable (Eric ran into this on Windows when he had a Pulumi program at the root of his file system. - Recusive symlinks leading to errors when trying to walk down the infinite chain. (See #1634 for one such example). The very frustrating thing about this is that when you hit an error its not clear what is going on and fixing it can be non-trivial. Even worse, in the normal case, all of these plugins are already installed and could be loaded by the host (in the common case, plugins are installed as a post install step when you run `npm install`) so if we simply didn't do this check at all, things would work great. This change does two things: 1. It does not stop at the first error we hit when discovering plugins, instead we just record the error and continue. 2. Does not fail the overall operation if there was an error. Instead, we return to the host what we have, which may be an incomplete view of the world. We glog the errors we did discover for diagnostics if we ever need them. I believe that long term most of this code gets deleted anyway. I expect we will move to a model long term where the engine faults in the plugin (downloading it if needed) when a request for the plugin arrives. But for now, we shouldn't block normal operations just because we couldn't answer a question with full fidelity. Fixes #1478
2018-08-08 22:48:36 +00:00
var allErrors *multierror.Error
for _, file := range files {
name := file.Name()
curr := filepath.Join(dir, name)
isDir := file.IsDir()
// if this is a symlink resolve it so our visitedPaths can track recursion
if (file.Type() & fs.ModeSymlink) != 0 {
2023-01-23 22:02:26 +00:00
symlink, err := filepath.EvalSymlinks(curr)
if err != nil {
allErrors = multierror.Append(allErrors, fmt.Errorf("resolving link in plugin dir %s: %w", curr, err))
continue
}
curr = symlink
// And re-stat the directory to get the resolved mode bits
fi, err := os.Stat(curr)
if err != nil {
allErrors = multierror.Append(allErrors, err)
continue
}
isDir = fi.IsDir()
}
if isDir {
// If a directory, recurse into it. However we have to take care to avoid recursing
// into nested policy packs. The plugins in a policy pack are not dependencies of the
// program, so we should not include them in the list of plugins to install.
policyPack, err := workspace.DetectPolicyPackPathFrom(curr)
if err != nil {
allErrors = multierror.Append(allErrors, err)
continue
}
policyPackDir := filepath.Dir(policyPack)
if policyPack != "" && strings.Contains(curr, policyPackDir) {
// The path is within a policy pack, stop recursing.
continue
}
more, err := getPluginsFromDir(
curr,
pulumiPackagePathToVersionMap,
inNodeModules || filepath.Base(dir) == "node_modules",
visitedPaths)
if err != nil {
Do not make errors during plugin discovery fatal The plugin host can ask the language host to provide a list of resource plugins that it thinks will be nessecary for use at deployment time, so they can be eagerly loaded. In NodeJS (the only language host that implements this RPC) This works by walking the directory tree rooted at the CWD of the project, looking for package.json files, parsing them and seeing it they have some marker property set. If they do, we add information about them which we return at the end of our walk. If there is *any* error, the entire operation fails. We've seen a bunch of cases where this happens: - Broken symlinks written by some editors as part of autosave. - Access denied errors when part of the tree is unwalkable (Eric ran into this on Windows when he had a Pulumi program at the root of his file system. - Recusive symlinks leading to errors when trying to walk down the infinite chain. (See #1634 for one such example). The very frustrating thing about this is that when you hit an error its not clear what is going on and fixing it can be non-trivial. Even worse, in the normal case, all of these plugins are already installed and could be loaded by the host (in the common case, plugins are installed as a post install step when you run `npm install`) so if we simply didn't do this check at all, things would work great. This change does two things: 1. It does not stop at the first error we hit when discovering plugins, instead we just record the error and continue. 2. Does not fail the overall operation if there was an error. Instead, we return to the host what we have, which may be an incomplete view of the world. We glog the errors we did discover for diagnostics if we ever need them. I believe that long term most of this code gets deleted anyway. I expect we will move to a model long term where the engine faults in the plugin (downloading it if needed) when a request for the plugin arrives. But for now, we shouldn't block normal operations just because we couldn't answer a question with full fidelity. Fixes #1478
2018-08-08 22:48:36 +00:00
allErrors = multierror.Append(allErrors, err)
}
// Even if there was an error, still append any plugins found in the dir.
plugins = append(plugins, more...)
} else if inNodeModules && name == "package.json" {
// if a package.json file within a node_modules package, parse it, and see if it's a source of plugins.
b, err := os.ReadFile(curr)
if err != nil {
2022-11-04 10:20:29 +00:00
allErrors = multierror.Append(allErrors, fmt.Errorf("reading package.json %s: %w", curr, err))
Do not make errors during plugin discovery fatal The plugin host can ask the language host to provide a list of resource plugins that it thinks will be nessecary for use at deployment time, so they can be eagerly loaded. In NodeJS (the only language host that implements this RPC) This works by walking the directory tree rooted at the CWD of the project, looking for package.json files, parsing them and seeing it they have some marker property set. If they do, we add information about them which we return at the end of our walk. If there is *any* error, the entire operation fails. We've seen a bunch of cases where this happens: - Broken symlinks written by some editors as part of autosave. - Access denied errors when part of the tree is unwalkable (Eric ran into this on Windows when he had a Pulumi program at the root of his file system. - Recusive symlinks leading to errors when trying to walk down the infinite chain. (See #1634 for one such example). The very frustrating thing about this is that when you hit an error its not clear what is going on and fixing it can be non-trivial. Even worse, in the normal case, all of these plugins are already installed and could be loaded by the host (in the common case, plugins are installed as a post install step when you run `npm install`) so if we simply didn't do this check at all, things would work great. This change does two things: 1. It does not stop at the first error we hit when discovering plugins, instead we just record the error and continue. 2. Does not fail the overall operation if there was an error. Instead, we return to the host what we have, which may be an incomplete view of the world. We glog the errors we did discover for diagnostics if we ever need them. I believe that long term most of this code gets deleted anyway. I expect we will move to a model long term where the engine faults in the plugin (downloading it if needed) when a request for the plugin arrives. But for now, we shouldn't block normal operations just because we couldn't answer a question with full fidelity. Fixes #1478
2018-08-08 22:48:36 +00:00
continue
}
var info packageJSON
if err := json.Unmarshal(b, &info); err != nil {
2022-11-04 10:20:29 +00:00
allErrors = multierror.Append(allErrors, fmt.Errorf("unmarshaling package.json %s: %w", curr, err))
continue
}
if info.Name == "@pulumi/pulumi" {
version, err := semver.Parse(info.Version)
if err != nil {
allErrors = multierror.Append(
2022-11-04 10:20:29 +00:00
allErrors, fmt.Errorf("Could not understand version %s in '%s': %w", info.Version, curr, err))
continue
}
pulumiPackagePathToVersionMap[curr] = version
}
ok, name, version, server, err := getPackageInfo(info)
if err != nil {
2022-11-04 10:20:29 +00:00
allErrors = multierror.Append(allErrors, fmt.Errorf("unmarshaling package.json %s: %w", curr, err))
} else if ok {
plugins = append(plugins, &pulumirpc.PluginDependency{
Name: name,
Kind: "resource",
Version: version,
Server: server,
})
}
}
}
Do not make errors during plugin discovery fatal The plugin host can ask the language host to provide a list of resource plugins that it thinks will be nessecary for use at deployment time, so they can be eagerly loaded. In NodeJS (the only language host that implements this RPC) This works by walking the directory tree rooted at the CWD of the project, looking for package.json files, parsing them and seeing it they have some marker property set. If they do, we add information about them which we return at the end of our walk. If there is *any* error, the entire operation fails. We've seen a bunch of cases where this happens: - Broken symlinks written by some editors as part of autosave. - Access denied errors when part of the tree is unwalkable (Eric ran into this on Windows when he had a Pulumi program at the root of his file system. - Recusive symlinks leading to errors when trying to walk down the infinite chain. (See #1634 for one such example). The very frustrating thing about this is that when you hit an error its not clear what is going on and fixing it can be non-trivial. Even worse, in the normal case, all of these plugins are already installed and could be loaded by the host (in the common case, plugins are installed as a post install step when you run `npm install`) so if we simply didn't do this check at all, things would work great. This change does two things: 1. It does not stop at the first error we hit when discovering plugins, instead we just record the error and continue. 2. Does not fail the overall operation if there was an error. Instead, we return to the host what we have, which may be an incomplete view of the world. We glog the errors we did discover for diagnostics if we ever need them. I believe that long term most of this code gets deleted anyway. I expect we will move to a model long term where the engine faults in the plugin (downloading it if needed) when a request for the plugin arrives. But for now, we shouldn't block normal operations just because we couldn't answer a question with full fidelity. Fixes #1478
2018-08-08 22:48:36 +00:00
return plugins, allErrors.ErrorOrNil()
}
// packageJSON is the minimal amount of package.json information we care about.
type packageJSON struct {
Name string `json:"name"`
Version string `json:"version"`
Pulumi plugin.PulumiPluginJSON `json:"pulumi"`
Main string `json:"main"`
Dependencies map[string]string `json:"dependencies"`
DevDependencies map[string]string `json:"devDependencies"`
}
// getPackageInfo returns a bool indicating whether the given package.json package has an associated Pulumi
2022-09-13 14:40:09 +00:00
// resource provider plugin. If it does, three strings are returned, the plugin name, and its semantic version and
// an optional server that can be used to download the plugin (this may be empty, in which case the "default" location
// should be used).
func getPackageInfo(info packageJSON) (bool, string, string, string, error) {
if info.Pulumi.Resource {
name, err := getPluginName(info)
if err != nil {
return false, "", "", "", err
}
version, err := getPluginVersion(info)
if err != nil {
return false, "", "", "", err
}
return true, name, version, info.Pulumi.Server, nil
}
return false, "", "", "", nil
}
// getPluginName takes a parsed package.json file and returns the corresponding Pulumi plugin name.
func getPluginName(info packageJSON) (string, error) {
// If it's specified in the "pulumi" section, return it as-is.
if info.Pulumi.Name != "" {
return info.Pulumi.Name, nil
}
2022-09-13 14:40:09 +00:00
// Otherwise, derive it from the top-level package name,
// only if it has @pulumi scope, otherwise fail.
name := info.Name
if name == "" {
return "", errors.New("missing expected \"name\" property")
}
2022-09-13 14:40:09 +00:00
// If the name has a @pulumi scope, we will just use its simple name. Otherwise, we use the fully scoped name.
// We do trim the leading @, however, since Pulumi resource providers do not use the same NPM convention.
if strings.Index(name, "@pulumi/") == 0 {
return name[strings.IndexRune(name, '/')+1:], nil
}
2022-09-13 14:40:09 +00:00
// if the package name does not start with @pulumi it means that it is a third-party package
// third-party packages _MUST_ have the plugin name in package.json in the pulumi section
// {
// "name": "@third-party-package"
// "pulumi": {
// "name": "<plugin name>"
// }
// }
2022-11-04 10:20:29 +00:00
return "", fmt.Errorf("Missing property \"name\" for the third-party plugin '%v' "+
2022-09-13 14:40:09 +00:00
"inside package.json under the \"pulumi\" section.", name)
}
// getPluginVersion takes a parsed package.json file and returns the semantic version of the Pulumi plugin.
func getPluginVersion(info packageJSON) (string, error) {
// See if it's specified in the "pulumi" section.
version := info.Pulumi.Version
if version == "" {
// If not, use the top-level package version.
version = info.Version
if version == "" {
return "", errors.New("Missing expected \"version\" property")
}
}
if strings.IndexRune(version, 'v') != 0 {
return "v" + version, nil
}
return version, nil
}
// When talking to the nodejs runtime we have three parties involved:
//
// Engine Monitor <==> Language Host (this code) <==> NodeJS sdk runtime.
//
// Instead of having the NodeJS sdk runtime communicating directly with the Engine Monitor, we
// instead have it communicate with us and we send all those messages to the real engine monitor
// itself. We do that by having ourselves launch our own grpc monitor server and passing the
// address of it to the NodeJS runtime. As far as the NodeJS sdk runtime is concerned, it is
// communicating directly with the engine.
//
// When NodeJS then communicates back with us over our server, we then forward the messages
// along untouched to the Engine Monitor. However, we also open an additional *non-grpc*
// channel to allow the sdk runtime to send us messages on. Specifically, this non-grpc channel
// is used entirely to allow the sdk runtime to make 'invoke' calls in a synchronous fashion.
// This is accomplished by avoiding grpc entirely (which has no facility for synchronous rpc
// calls), and instead operating over a pair of files coordinated between us and the sdk
// runtime. One file is used by it to send us messages, and one file is used by us to send
// messages back. Because these are just files, nodejs natively supports allowing both sides to
// read and write from each synchronously.
//
// When we receive the sync-invoke messages from the nodejs sdk we deserialize things off of the
// file and then make a synchronous call to the real engine `invoke` monitor endpoint. Unlike
// nodejs, we have no problem calling this synchronously, and can block until we get the
// response which we can then synchronously send to nodejs.
// Run is the RPC endpoint for LanguageRuntimeServer::Run
func (host *nodeLanguageHost) Run(ctx context.Context, req *pulumirpc.RunRequest) (*pulumirpc.RunResponse, error) {
tracingSpan := opentracing.SpanFromContext(ctx)
// Make a connection to the real monitor that we will forward messages to.
2020-04-20 22:25:51 +00:00
conn, err := grpc.Dial(
req.GetMonitorAddress(),
grpc.WithTransportCredentials(insecure.NewCredentials()),
2020-04-20 22:25:51 +00:00
rpcutil.GrpcChannelOptions(),
)
if err != nil {
return nil, err
}
// Make a client around that connection. We can then make our own server that will act as a
// monitor for the sdk and forward to the real monitor.
target := pulumirpc.NewResourceMonitorClient(conn)
// Channel to control the server lifetime. Once `Run` finishes, we'll shutdown the server.
serverCancel := make(chan bool)
defer func() {
serverCancel <- true
close(serverCancel)
}()
// Launch the rpc server giving it the real monitor to forward messages to.
2022-11-01 15:15:09 +00:00
handle, err := rpcutil.ServeWithOptions(rpcutil.ServeOptions{
Cancel: serverCancel,
Init: func(srv *grpc.Server) error {
pulumirpc.RegisterResourceMonitorServer(srv, &monitorProxy{target: target})
return nil
},
2022-11-01 15:15:09 +00:00
Options: rpcutil.OpenTracingServerInterceptorOptions(tracingSpan),
})
if err != nil {
return nil, err
}
// Create the pipes we'll use to communicate synchronously with the nodejs process. Once we're
// done using the pipes clean them up so we don't leave anything around in the user file system.
pipes, pipesDone, err := createAndServePipes(ctx, target)
if err != nil {
return nil, err
}
defer pipes.shutdown()
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
nodeBin, err := exec.LookPath("node")
if err != nil {
Return error in RPC response instead of exiting in the language runtime (#16162) # Description If we can't start the node process for some reason, report back the error via RPC instead of exiting immediately. Fixes https://github.com/pulumi/pulumi/issues/11706 Instead of: <img width="1018" alt="Screenshot 2024-05-10 at 15 05 37" src="https://github.com/pulumi/pulumi/assets/387068/c418e890-34a3-473d-bbb5-6d45679f8bb6"> we get <img width="1294" alt="Screenshot 2024-05-10 at 15 06 19" src="https://github.com/pulumi/pulumi/assets/387068/2ef1b0af-159d-4e99-9a6c-9c6b21375ea8"> ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-05-10 15:16:13 +00:00
return &pulumirpc.RunResponse{Error: "could not find node on the $PATH: " + err.Error()}, nil
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
}
runPath := os.Getenv("PULUMI_LANGUAGE_NODEJS_RUN_PATH")
if runPath == "" {
runPath = defaultRunPath
}
Run conformance tests with and without ts-node (#15809) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/15075. This doubles up the conformance tests to run with `tsc` instead of `ts-node`. Getting this to work in the conformance framework required a bit of pushing the envelope for what it means to "install dependencies" and "run", but it works for testing, and isn't horribly exposed to users. It also points out a couple of things that we might want to clean up for users wanting the same workflow. 1. Adding a `Build` command (which we might want for the type checker work added in https://github.com/pulumi/pulumi/pull/15725 as well). 2. If you set "main" to ./bin because you're using tsc, you can't then use InstallDependencies because to start with the "bin" folder is missing. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-03-28 15:44:25 +00:00
// If we're forcing tsc the program directory for running is actually ./bin
programDirectory := req.Info.ProgramDirectory
if host.forceTsc {
req.Info.ProgramDirectory = filepath.Join(programDirectory, "bin")
}
runPath, err = locateModule(ctx, runPath, programDirectory, nodeBin)
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
if err != nil {
Return error in RPC response instead of exiting in the language runtime (#16162) # Description If we can't start the node process for some reason, report back the error via RPC instead of exiting immediately. Fixes https://github.com/pulumi/pulumi/issues/11706 Instead of: <img width="1018" alt="Screenshot 2024-05-10 at 15 05 37" src="https://github.com/pulumi/pulumi/assets/387068/c418e890-34a3-473d-bbb5-6d45679f8bb6"> we get <img width="1294" alt="Screenshot 2024-05-10 at 15 06 19" src="https://github.com/pulumi/pulumi/assets/387068/2ef1b0af-159d-4e99-9a6c-9c6b21375ea8"> ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-05-10 15:16:13 +00:00
return &pulumirpc.RunResponse{Error: err.Error()}, nil
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
}
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
// Channel producing the final response we want to issue to our caller. Will get the result of
// the actual nodejs process we launch, or any results caused by errors in our server/pipes.
responseChannel := make(chan *pulumirpc.RunResponse)
// now, launch the nodejs process and actually run the user code in it.
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
go func() {
defer close(responseChannel)
responseChannel <- host.execNodejs(
ctx, req, nodeBin, runPath,
fmt.Sprintf("127.0.0.1:%d", handle.Port), pipes.directory())
}()
// Wait for one of our launched goroutines to signal that we're done. This might be our proxy
// (in the case of errors), or the launched nodejs completing (either successfully, or with
// errors).
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
for {
select {
case err := <-handle.Done:
if err != nil {
return &pulumirpc.RunResponse{Error: err.Error()}, nil
}
case err := <-pipesDone:
if err != nil {
return &pulumirpc.RunResponse{Error: err.Error()}, nil
}
case response := <-responseChannel:
return response, nil
}
}
}
// Launch the nodejs process and wait for it to complete. Report success or any errors using the
// `responseChannel` arg.
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
func (host *nodeLanguageHost) execNodejs(ctx context.Context, req *pulumirpc.RunRequest,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
nodeBin, runPath, address, pipesDirectory string,
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
) *pulumirpc.RunResponse {
// Actually launch nodejs and process the result of it into an appropriate response object.
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
args := host.constructArguments(req, runPath, address, pipesDirectory)
config, err := host.constructConfig(req)
if err != nil {
err = fmt.Errorf("failed to serialize configuration: %w", err)
return &pulumirpc.RunResponse{Error: err.Error()}
}
configSecretKeys, err := host.constructConfigSecretKeys(req)
if err != nil {
err = fmt.Errorf("failed to serialize configuration secret keys: %w", err)
return &pulumirpc.RunResponse{Error: err.Error()}
}
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
env := os.Environ()
env = append(env, pulumiConfigVar+"="+config)
env = append(env, pulumiConfigSecretKeysVar+"="+configSecretKeys)
opts, err := parseOptions(req.Info.Options.AsMap())
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
if err != nil {
return &pulumirpc.RunResponse{Error: err.Error()}
}
if opts.typescript {
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
env = append(env, "PULUMI_NODEJS_TYPESCRIPT=true")
}
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
if opts.tsconfigpath != "" {
env = append(env, "PULUMI_NODEJS_TSCONFIG_PATH="+opts.tsconfigpath)
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
}
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
nodeargs, err := shlex.Split(opts.nodeargs)
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
if err != nil {
return &pulumirpc.RunResponse{Error: err.Error()}
}
nodeargs = append(nodeargs, args...)
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
if logging.V(5) {
commandStr := strings.Join(nodeargs, " ")
logging.V(5).Infoln("Language host launching process: ", nodeBin, commandStr)
}
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
// Now simply spawn a process to execute the requested program, wiring up stdout/stderr directly.
var errResult string
// #nosec G204
cmd := exec.Command(nodeBin, nodeargs...)
// Copy cmd.Stdout to os.Stdout. Nodejs sometimes changes the blocking mode of its stdout/stderr,
// so it's unsafe to assign cmd.Stdout directly to os.Stdout. See the description of
// `runWithOutput` for more details.
cmd.Stdout = struct{ io.Writer }{os.Stdout}
r, w := io.Pipe()
cmd.Stderr = w
// Get a duplicate reader of stderr so that we can both scan it and write it to os.Stderr.
stderrDup := io.TeeReader(r, os.Stderr)
sniffer := newOOMSniffer()
sniffer.Scan(stderrDup)
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
cmd.Env = env
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
tracingSpan, _ := opentracing.StartSpanFromContext(ctx,
"execNodejs",
opentracing.Tag{Key: "component", Value: "exec.Command"},
opentracing.Tag{Key: "command", Value: nodeBin},
opentracing.Tag{Key: "args", Value: nodeargs})
defer tracingSpan.Finish()
if err := cmd.Run(); err != nil {
// NodeJS stdout is complicated enough that we should explicitly flush stdout and stderr here. NodeJS does
// process writes using console.out and console.err synchronously, but it does not process writes using
// `process.stdout.write` or `process.stderr.write` synchronously, and it is possible that there exist unflushed
// writes on those file descriptors at the time that the Node process exits.
//
// Because of this, we explicitly flush stdout and stderr so that we are absolutely sure that we capture any
// error messages in the engine.
contract.IgnoreError(os.Stdout.Sync())
contract.IgnoreError(os.Stderr.Sync())
if exiterr, ok := err.(*exec.ExitError); ok {
// If the program ran, but exited with a non-zero error code. This will happen often,
// since user errors will trigger this. So, the error message should look as nice as
// possible.
switch code := exiterr.ExitCode(); code {
case 0:
// This really shouldn't happen, but if it does, we don't want to render "non-zero exit code"
err = fmt.Errorf("Program exited unexpectedly: %w", exiterr)
case nodeJSProcessExitedAfterShowingUserActionableMessage:
// Check if we got special exit code that means "we already gave the user an
// actionable message". In that case, we can simply bail out and terminate `pulumi`
// without showing any more messages.
return &pulumirpc.RunResponse{Error: "", Bail: true}
default:
err = fmt.Errorf("Program exited with non-zero exit code: %d", code)
sniffer.Wait()
if sniffer.Detected() {
err = fmt.Errorf("Program exited with non-zero exit code: %d. %s", code, sniffer.Message())
}
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
}
} else {
// Otherwise, we didn't even get to run the program. This ought to never happen unless there's
// a bug or system condition that prevented us from running the language exec. Issue a scarier error.
err = fmt.Errorf("Problem executing program (could not run language executor): %w", err)
}
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
errResult = err.Error()
}
// notify our caller of the response we got from the nodejs process. Note: this is done
// unilaterally. this is how we signal to nodeLanguageHost.Run that we are done and it can
// return to its caller.
Change nodejs waiting on pipes to not panic (#13689) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Hit this while testing with matrix testing. Fairly frequently the go routine waiting for `pipesDone` would try to write to the response channel and panic because it was already closed. This rewrites the method to only use the response channel for the running program, and we just explictly wait on all three events (pipes, handle, program) and return the first one that triggers. Also it refactors execNodejs to just be a sync method that returns the response. The management of that running in a goroutine and writing to a channel is just kept in Run. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works - Current tests should cover current use of this, and matrix testing will verify that this fix is in place for that. <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-09-03 07:26:15 +00:00
return &pulumirpc.RunResponse{Error: errResult}
}
// constructArguments constructs a command-line for `pulumi-language-nodejs`
// by enumerating all of the optional and non-optional arguments present
// in a RunRequest.
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
func (host *nodeLanguageHost) constructArguments(
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
req *pulumirpc.RunRequest, runPath, address, pipesDirectory string,
) []string {
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
args := []string{runPath}
maybeAppendArg := func(k, v string) {
if v != "" {
args = append(args, "--"+k, v)
}
}
maybeAppendArg("monitor", address)
maybeAppendArg("engine", host.engineAddress)
maybeAppendArg("sync", pipesDirectory)
maybeAppendArg("organization", req.GetOrganization())
maybeAppendArg("project", req.GetProject())
maybeAppendArg("stack", req.GetStack())
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
maybeAppendArg("pwd", req.Info.ProgramDirectory)
if req.GetDryRun() {
args = append(args, "--dry-run")
}
Enable perfsprint linter (#14813) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Prompted by a comment in another review: https://github.com/pulumi/pulumi/pull/14654#discussion_r1419995945 This lints that we don't use `fmt.Errorf` when `errors.New` will suffice, it also covers a load of other cases where `Sprintf` is sub-optimal. Most of these edits were made by running `perfsprint --fix`. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-12 12:19:42 +00:00
maybeAppendArg("query-mode", strconv.FormatBool(req.GetQueryMode()))
maybeAppendArg("parallel", strconv.Itoa(int(req.GetParallel())))
maybeAppendArg("tracing", host.tracing)
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
// The engine should always pass a name for entry point, even if its just "." for the program directory.
args = append(args, req.Info.EntryPoint)
args = append(args, req.GetArgs()...)
return args
}
// constructConfig JSON-serializes the configuration data given as part of
// a RunRequest.
func (host *nodeLanguageHost) constructConfig(req *pulumirpc.RunRequest) (string, error) {
configMap := req.GetConfig()
if configMap == nil {
return "{}", nil
}
// While we transition from the old format for config keys (<package>:config:<name> to <package>:<name>), we want
// to support the newest version of the langhost running older packages, so the config bag we present to them looks
// like the old world. Newer versions of the @pulumi/pulumi package handle both formats and when we stop supporting
// older versions, we can remove this code.
transformedConfig := make(map[string]string, len(configMap))
for k, v := range configMap {
pk, err := config.ParseKey(k)
if err != nil {
return "", err
}
transformedConfig[pk.Namespace()+":config:"+pk.Name()] = v
}
configJSON, err := json.Marshal(transformedConfig)
if err != nil {
return "", err
}
return string(configJSON), nil
}
// constructConfigSecretKeys JSON-serializes the list of keys that contain secret values given as part of
// a RunRequest.
func (host *nodeLanguageHost) constructConfigSecretKeys(req *pulumirpc.RunRequest) (string, error) {
configSecretKeys := req.GetConfigSecretKeys()
if configSecretKeys == nil {
return "[]", nil
}
configSecretKeysJSON, err := json.Marshal(configSecretKeys)
if err != nil {
return "", err
}
return string(configSecretKeysJSON), nil
}
Remove deprecated Protobufs imports (#15158) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> github.com/golang/protobuf is marked deprecated and I was getting increasingly triggered by the inconsistency of importing the `Empty` type from "github.com/golang/protobuf/ptypes/empty" or "google.golang.org/protobuf/types/known/emptypb" as "pbempty" or "empty" or "emptypb". Similar for the struct type. So this replaces all the Protobufs imports with ones from "google.golang.org/protobuf", normalises the import name to always just be the module name (emptypb), and adds the depguard linter to ensure we don't use the deprecated package anymore. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-01-17 09:35:20 +00:00
func (host *nodeLanguageHost) GetPluginInfo(ctx context.Context, req *emptypb.Empty) (*pulumirpc.PluginInfo, error) {
return &pulumirpc.PluginInfo{
Version: version.Version,
}, nil
}
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
func (host *nodeLanguageHost) InstallDependencies(
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
req *pulumirpc.InstallDependenciesRequest, server pulumirpc.LanguageRuntime_InstallDependenciesServer,
) error {
closer, stdout, stderr, err := rpcutil.MakeInstallDependenciesStreams(server, req.IsTerminal)
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
if err != nil {
return err
}
// best effort close, but we try an explicit close and error check at the end as well
defer closer.Close()
tracingSpan, ctx := opentracing.StartSpanFromContext(server.Context(), "npm-install")
defer tracingSpan.Finish()
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
stdout.Write([]byte("Installing dependencies...\n\n"))
workspaceRoot := req.Info.ProgramDirectory
newWorkspaceRoot, err := npm.FindWorkspaceRoot(req.Info.ProgramDirectory)
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
if err != nil {
// If we are not in a npm/yarn workspace, we will keep the current directory as the root.
if !errors.Is(err, npm.ErrNotInWorkspace) {
return fmt.Errorf("failure while trying to find workspace root: %w", err)
}
} else {
stdout.Write([]byte(fmt.Sprintf("Detected workspace root at %s\n", newWorkspaceRoot)))
workspaceRoot = newWorkspaceRoot
}
opts, err := parseOptions(req.Info.Options.AsMap())
if err != nil {
return fmt.Errorf("failed to parse options: %w", err)
}
_, err = npm.Install(ctx, opts.packagemanager, workspaceRoot, false /*production*/, stdout, stderr)
if err != nil {
return err
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
}
stdout.Write([]byte("Finished installing dependencies\n\n"))
Run conformance tests with and without ts-node (#15809) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/15075. This doubles up the conformance tests to run with `tsc` instead of `ts-node`. Getting this to work in the conformance framework required a bit of pushing the envelope for what it means to "install dependencies" and "run", but it works for testing, and isn't horribly exposed to users. It also points out a couple of things that we might want to clean up for users wanting the same workflow. 1. Adding a `Build` command (which we might want for the type checker work added in https://github.com/pulumi/pulumi/pull/15725 as well). 2. If you set "main" to ./bin because you're using tsc, you can't then use InstallDependencies because to start with the "bin" folder is missing. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-03-28 15:44:25 +00:00
if host.forceTsc {
// If we're forcing tsc for conformance testing this is our chance to run it before actually running the program.
// We probably want to see about making something like this an explicit "pulumi build" step, but for now shim'ing this
// in here works well enough for conformance testing.
tscCmd := exec.Command("npx", "tsc")
tscCmd.Dir = req.Info.ProgramDirectory
if err := runWithOutput(tscCmd, os.Stdout, os.Stderr); err != nil {
Run conformance tests with and without ts-node (#15809) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/15075. This doubles up the conformance tests to run with `tsc` instead of `ts-node`. Getting this to work in the conformance framework required a bit of pushing the envelope for what it means to "install dependencies" and "run", but it works for testing, and isn't horribly exposed to users. It also points out a couple of things that we might want to clean up for users wanting the same workflow. 1. Adding a `Build` command (which we might want for the type checker work added in https://github.com/pulumi/pulumi/pull/15725 as well). 2. If you set "main" to ./bin because you're using tsc, you can't then use InstallDependencies because to start with the "bin" folder is missing. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-03-28 15:44:25 +00:00
return fmt.Errorf("failed to run tsc: %w", err)
}
}
all: Fix revive issues Fixes the following issues found by revive included in the latest release of golangci-lint. Full list of issues: **pkg** ``` backend/display/object_diff.go:47:10: superfluous-else: if block ends with a break statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive) backend/display/object_diff.go:716:12: redefines-builtin-id: redefinition of the built-in function delete (revive) backend/display/object_diff.go:742:14: redefines-builtin-id: redefinition of the built-in function delete (revive) backend/display/object_diff.go:983:10: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (revive) backend/httpstate/backend.go:1814:4: redefines-builtin-id: redefinition of the built-in function cap (revive) backend/httpstate/backend.go:1824:5: redefines-builtin-id: redefinition of the built-in function cap (revive) backend/httpstate/client/client.go:444:2: if-return: redundant if ...; err != nil check, just return error instead. (revive) backend/httpstate/client/client.go:455:2: if-return: redundant if ...; err != nil check, just return error instead. (revive) cmd/pulumi/org.go:113:4: if-return: redundant if ...; err != nil check, just return error instead. (revive) cmd/pulumi/util.go:216:2: if-return: redundant if ...; err != nil check, just return error instead. (revive) codegen/docs/gen.go:428:2: redefines-builtin-id: redefinition of the built-in function copy (revive) codegen/hcl2/model/expression.go:2151:5: redefines-builtin-id: redefinition of the built-in function close (revive) codegen/hcl2/syntax/comments.go:151:2: redefines-builtin-id: redefinition of the built-in function close (revive) codegen/hcl2/syntax/comments.go:329:3: redefines-builtin-id: redefinition of the built-in function close (revive) codegen/hcl2/syntax/comments.go:381:5: redefines-builtin-id: redefinition of the built-in function close (revive) codegen/nodejs/gen.go:1367:5: redefines-builtin-id: redefinition of the built-in function copy (revive) codegen/python/gen_program_expressions.go:136:2: redefines-builtin-id: redefinition of the built-in function close (revive) codegen/python/gen_program_expressions.go:142:3: redefines-builtin-id: redefinition of the built-in function close (revive) codegen/report/report.go:126:6: redefines-builtin-id: redefinition of the built-in function panic (revive) codegen/schema/docs_test.go:210:10: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive) codegen/schema/schema.go:790:2: redefines-builtin-id: redefinition of the built-in type any (revive) codegen/schema/schema.go:793:4: redefines-builtin-id: redefinition of the built-in type any (revive) resource/deploy/plan.go:506:2: if-return: redundant if ...; err != nil check, just return error instead. (revive) resource/deploy/snapshot_test.go:59:3: redefines-builtin-id: redefinition of the built-in function copy (revive) resource/deploy/state_builder.go:108:2: redefines-builtin-id: redefinition of the built-in function copy (revive) ``` **sdk** ``` go/common/resource/plugin/context.go:142:2: redefines-builtin-id: redefinition of the built-in function copy (revive) go/common/resource/plugin/plugin.go:142:12: superfluous-else: if block ends with a break statement, so drop this else and outdent its block (revive) go/common/resource/properties_diff.go:114:2: redefines-builtin-id: redefinition of the built-in function len (revive) go/common/resource/properties_diff.go:117:4: redefines-builtin-id: redefinition of the built-in function len (revive) go/common/resource/properties_diff.go:122:4: redefines-builtin-id: redefinition of the built-in function len (revive) go/common/resource/properties_diff.go:127:4: redefines-builtin-id: redefinition of the built-in function len (revive) go/common/resource/properties_diff.go:132:4: redefines-builtin-id: redefinition of the built-in function len (revive) go/common/util/deepcopy/copy.go:30:1: redefines-builtin-id: redefinition of the built-in function copy (revive) go/common/workspace/creds.go:242:2: if-return: redundant if ...; err != nil check, just return error instead. (revive) go/pulumi-language-go/main.go:569:2: if-return: redundant if ...; err != nil check, just return error instead. (revive) go/pulumi-language-go/main.go:706:2: if-return: redundant if ...; err != nil check, just return error instead. (revive) go/pulumi/run_test.go:925:2: redefines-builtin-id: redefinition of the built-in type any (revive) go/pulumi/run_test.go:933:3: redefines-builtin-id: redefinition of the built-in type any (revive) nodejs/cmd/pulumi-language-nodejs/main.go:778:2: if-return: redundant if ...; err != nil check, just return error instead. (revive) python/cmd/pulumi-language-python/main.go:1011:2: if-return: redundant if ...; err != nil check, just return error instead. (revive) python/cmd/pulumi-language-python/main.go:863:2: if-return: redundant if ...; err != nil check, just return error instead. (revive) python/python.go:230:2: redefines-builtin-id: redefinition of the built-in function print (revive) ``` **tests** ``` integration/integration_util_test.go:282:11: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive) ```
2023-03-20 23:48:02 +00:00
return closer.Close()
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
}
func (host *nodeLanguageHost) RuntimeOptionsPrompts(ctx context.Context,
req *pulumirpc.RuntimeOptionsRequest,
) (*pulumirpc.RuntimeOptionsResponse, error) {
var prompts []*pulumirpc.RuntimeOptionPrompt
rawOpts := req.Info.Options.AsMap()
if _, hasPackagemanager := rawOpts["packagemanager"]; !hasPackagemanager {
prompts = append(prompts, &pulumirpc.RuntimeOptionPrompt{
Key: "packagemanager",
Description: "The package manager to use for installing dependencies",
PromptType: pulumirpc.RuntimeOptionPrompt_STRING,
Choices: plugin.MakeExecutablePromptChoices("npm", "pnpm", "yarn"),
Default: &pulumirpc.RuntimeOptionPrompt_RuntimeOptionValue{
PromptType: pulumirpc.RuntimeOptionPrompt_STRING,
StringValue: "npm",
},
})
}
return &pulumirpc.RuntimeOptionsResponse{
Prompts: prompts,
}, nil
}
func (host *nodeLanguageHost) About(ctx context.Context,
req *pulumirpc.AboutRequest,
) (*pulumirpc.AboutResponse, error) {
getResponse := func(execString string, args ...string) (string, string, error) {
ex, err := executable.FindExecutable(execString)
if err != nil {
return "", "", fmt.Errorf("could not find executable '%s': %w", execString, err)
}
cmd := exec.Command(ex, args...)
var out []byte
if out, err = cmd.Output(); err != nil {
cmd := ex
if len(args) != 0 {
cmd += " " + strings.Join(args, " ")
}
return "", "", fmt.Errorf("failed to execute '%s'", cmd)
}
return ex, strings.TrimSpace(string(out)), nil
}
node, version, err := getResponse("node", "--version")
if err != nil {
return nil, err
}
return &pulumirpc.AboutResponse{
Executable: node,
Version: version,
}, nil
}
// The shape of a `yarn list --json`'s output.
type yarnLock struct {
Type string `json:"type"`
Data yarnLockData `json:"data"`
}
type yarnLockData struct {
Type string `json:"type"`
Trees []yarnLockTree `json:"trees"`
}
type yarnLockTree struct {
Name string `json:"name"`
Children []yarnLockTree `json:"children"`
}
Test GetDependencies and library versions in conformance testing (#15324) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This adds testing to the language conformance test command to check that GetProgramDependencies returns the expected data. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-02-06 12:38:44 +00:00
func parseYarnLockFile(programDirectory, path string) ([]*pulumirpc.DependencyInfo, error) {
ex, err := executable.FindExecutable("yarn")
if err != nil {
return nil, fmt.Errorf("found %s but no yarn executable: %w", path, err)
}
cmdArgs := []string{"list", "--json"}
cmd := exec.Command(ex, cmdArgs...)
Test GetDependencies and library versions in conformance testing (#15324) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This adds testing to the language conformance test command to check that GetProgramDependencies returns the expected data. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-02-06 12:38:44 +00:00
cmd.Dir = programDirectory
out, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("failed to run \"%s %s\": %w", ex, strings.Join(cmdArgs, " "), err)
}
var lock yarnLock
if err = json.Unmarshal(out, &lock); err != nil {
return nil, fmt.Errorf("failed to parse\"%s %s\": %w", ex, strings.Join(cmdArgs, " "), err)
}
leafs := lock.Data.Trees
result := make([]*pulumirpc.DependencyInfo, len(leafs))
// Has the form name@version
splitName := func(index int, nameVersion string) (string, string, error) {
if nameVersion == "" {
return "", "", fmt.Errorf("expected \"name\" in dependency %d", index)
}
split := strings.LastIndex(nameVersion, "@")
if split == -1 {
return "", "", fmt.Errorf("failed to parse name and version from %s", nameVersion)
}
return nameVersion[:split], nameVersion[split+1:], nil
}
for i, v := range leafs {
name, version, err := splitName(i, v.Name)
if err != nil {
return nil, err
}
result[i] = &pulumirpc.DependencyInfo{
Name: name,
Version: version,
}
}
return result, nil
}
// Describes the shape of `npm ls --json --depth=0`'s output.
type npmFile struct {
Name string `json:"name"`
LockFileVersion int `json:"lockfileVersion"`
Requires bool `json:"requires"`
Dependencies map[string]npmPackage `json:"dependencies"`
}
// A package in npmFile.
type npmPackage struct {
Version string `json:"version"`
Resolved string `json:"resolved"`
}
Test GetDependencies and library versions in conformance testing (#15324) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This adds testing to the language conformance test command to check that GetProgramDependencies returns the expected data. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-02-06 12:38:44 +00:00
func parseNpmLockFile(programDirectory, path string) ([]*pulumirpc.DependencyInfo, error) {
ex, err := executable.FindExecutable("npm")
if err != nil {
return nil, fmt.Errorf("found %s but not npm: %w", path, err)
}
cmdArgs := []string{"ls", "--json", "--depth=0"}
cmd := exec.Command(ex, cmdArgs...)
Test GetDependencies and library versions in conformance testing (#15324) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This adds testing to the language conformance test command to check that GetProgramDependencies returns the expected data. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-02-06 12:38:44 +00:00
cmd.Dir = programDirectory
out, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf(`failed to run "%s %s": %w`, ex, strings.Join(cmdArgs, " "), err)
}
file := npmFile{}
if err = json.Unmarshal(out, &file); err != nil {
return nil, fmt.Errorf(`failed to parse \"%s %s": %w`, ex, strings.Join(cmdArgs, " "), err)
}
result := make([]*pulumirpc.DependencyInfo, len(file.Dependencies))
var i int
for k, v := range file.Dependencies {
result[i] = &pulumirpc.DependencyInfo{
Name: k,
Version: v.Version,
}
i++
}
return result, nil
}
// Intersect a list of packages with the contents of `package.json`. Returns
// only packages that appear in both sets. `path` is used only for error handling.
func crossCheckPackageJSONFile(path string, file []byte,
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
packages []*pulumirpc.DependencyInfo,
) ([]*pulumirpc.DependencyInfo, error) {
var body packageJSON
if err := json.Unmarshal(file, &body); err != nil {
return nil, fmt.Errorf("could not parse %s: %w", path, err)
}
dependencies := make(map[string]string)
for k, v := range body.Dependencies {
dependencies[k] = v
}
for k, v := range body.DevDependencies {
dependencies[k] = v
}
// There should be 1 (& only 1) instantiated dependency for each
// dependency in package.json. We do this because we want to get the
// actual version (not the range) that exists in lock files.
result := make([]*pulumirpc.DependencyInfo, len(dependencies))
i := 0
for _, v := range packages {
if _, exists := dependencies[v.Name]; exists {
result[i] = v
// Some direct dependencies are also transitive dependencies. We
// only want to grab them once.
delete(dependencies, v.Name)
i++
}
}
return result, nil
}
func (host *nodeLanguageHost) GetProgramDependencies(
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
ctx context.Context, req *pulumirpc.GetProgramDependenciesRequest,
) (*pulumirpc.GetProgramDependenciesResponse, error) {
// We get the node dependencies. This requires either a yarn.lock file and the
// yarn executable, a package-lock.json file and the npm executable. If
// transitive is false, we also need the package.json file.
//
// If we find a yarn.lock file, we assume that yarn is used.
// Only then do we look for a package-lock.json file.
// Neither "yarn list" or "npm ls" can describe what packages are required
//
// (direct dependencies). Only what packages they have installed (transitive
// dependencies). This means that to accurately report only direct
// dependencies, we need to also parse "package.json" and intersect it with
// reported dependencies.
var err error
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
yarnFile := filepath.Join(req.Info.ProgramDirectory, "yarn.lock")
npmFile := filepath.Join(req.Info.ProgramDirectory, "package-lock.json")
packageFile := filepath.Join(req.Info.ProgramDirectory, "package.json")
var result []*pulumirpc.DependencyInfo
if _, err = os.Stat(yarnFile); err == nil {
Test GetDependencies and library versions in conformance testing (#15324) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This adds testing to the language conformance test command to check that GetProgramDependencies returns the expected data. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-02-06 12:38:44 +00:00
result, err = parseYarnLockFile(req.Info.ProgramDirectory, yarnFile)
if err != nil {
return nil, err
}
} else if _, err = os.Stat(npmFile); err == nil {
Test GetDependencies and library versions in conformance testing (#15324) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This adds testing to the language conformance test command to check that GetProgramDependencies returns the expected data. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-02-06 12:38:44 +00:00
result, err = parseNpmLockFile(req.Info.ProgramDirectory, npmFile)
if err != nil {
return nil, err
}
} else if os.IsNotExist(err) {
return nil, fmt.Errorf("could not find either %s or %s", yarnFile, npmFile)
} else {
return nil, fmt.Errorf("could not get node dependency data: %w", err)
}
if !req.TransitiveDependencies {
file, err := os.ReadFile(packageFile)
if os.IsNotExist(err) {
return nil, fmt.Errorf("could not find %s. "+
"Please include this in your report and run "+
`pulumi about --transitive" to get a list of used packages`,
packageFile)
} else if err != nil {
return nil, fmt.Errorf("could not read %s: %w", packageFile, err)
}
result, err = crossCheckPackageJSONFile(packageFile, file, result)
if err != nil {
return nil, err
}
}
return &pulumirpc.GetProgramDependenciesResponse{
Dependencies: result,
}, nil
}
func (host *nodeLanguageHost) RunPlugin(
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
req *pulumirpc.RunPluginRequest, server pulumirpc.LanguageRuntime_RunPluginServer,
) error {
return errors.New("not supported")
}
2023-05-12 13:38:36 +00:00
func (host *nodeLanguageHost) GenerateProject(
ctx context.Context, req *pulumirpc.GenerateProjectRequest,
) (*pulumirpc.GenerateProjectResponse, error) {
loader, err := schema.NewLoaderClient(req.LoaderTarget)
2023-05-12 13:38:36 +00:00
if err != nil {
return nil, err
}
var extraOptions []pcl.BindOption
if !req.Strict {
extraOptions = append(extraOptions, pcl.NonStrictBindOptions()...)
}
// for nodejs, prefer output-versioned invokes
extraOptions = append(extraOptions, pcl.PreferOutputVersionedInvokes)
program, diags, err := pcl.BindDirectory(req.SourceDirectory, loader, extraOptions...)
2023-05-12 13:38:36 +00:00
if err != nil {
return nil, err
}
2023-05-26 10:32:19 +00:00
if diags.HasErrors() {
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
2023-05-26 10:32:19 +00:00
return &pulumirpc.GenerateProjectResponse{
Diagnostics: rpcDiagnostics,
}, nil
2023-05-12 13:38:36 +00:00
}
var project workspace.Project
if err := json.Unmarshal([]byte(req.Project), &project); err != nil {
return nil, err
}
Run conformance tests with and without ts-node (#15809) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/15075. This doubles up the conformance tests to run with `tsc` instead of `ts-node`. Getting this to work in the conformance framework required a bit of pushing the envelope for what it means to "install dependencies" and "run", but it works for testing, and isn't horribly exposed to users. It also points out a couple of things that we might want to clean up for users wanting the same workflow. 1. Adding a `Build` command (which we might want for the type checker work added in https://github.com/pulumi/pulumi/pull/15725 as well). 2. If you set "main" to ./bin because you're using tsc, you can't then use InstallDependencies because to start with the "bin" folder is missing. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-03-28 15:44:25 +00:00
err = codegen.GenerateProject(req.TargetDirectory, project, program, req.LocalDependencies, host.forceTsc)
2023-05-12 13:38:36 +00:00
if err != nil {
return nil, err
}
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
2023-05-26 10:32:19 +00:00
return &pulumirpc.GenerateProjectResponse{
Diagnostics: rpcDiagnostics,
}, nil
2023-05-12 13:38:36 +00:00
}
func (host *nodeLanguageHost) GenerateProgram(
ctx context.Context, req *pulumirpc.GenerateProgramRequest,
) (*pulumirpc.GenerateProgramResponse, error) {
loader, err := schema.NewLoaderClient(req.LoaderTarget)
2023-05-12 13:38:36 +00:00
if err != nil {
return nil, err
}
parser := hclsyntax.NewParser()
// Load all .pp files in the directory
for path, contents := range req.Source {
err = parser.ParseFile(strings.NewReader(contents), path)
if err != nil {
return nil, err
}
diags := parser.Diagnostics
if diags.HasErrors() {
return nil, diags
}
}
[cli/import] Fix undefined variable errors in code generation when imported resources use a parent or provider (#16786) Fixes #15410 Fixes #13339 ## Problem Context When using `pulumi import` we generate code snippets for the resources that were imported. Sometimes the user specifies `--parent parentName=URN` or `--provider providerName=URN` which tweak the parent or provider that the imported resources uses. When using `--parent` or `--provider` the generated code emits a resource option `parent = parentName` (in case of using `--parent`) where `parentName` is an unbound variable. Usually unbound variables would result in a _bind_ error such as `error: undefined variable parentName` when type-checking the program however in the import code generation we specify the bind option `pcl.AllowMissingVariables` which turns that unbound variable errors into warnings and code generation can continue to emit code. This is all good and works as expected. However in the issues linked above, we do get an _error_ for unbound variables in generated code even though we specified `AllowMissingVariables`. The problem as it turns out is when we are trying to generate code via dynamically loaded `LangaugeRuntime` plugins. Specifically for NodeJS and Python, we load `pulumi-language-nodejs` or `pulumi-language-python` and call `GenerateProgram` to get the generated program. That function `GenerateProgram` takes the text _SOURCE_ of the a bound program (one that was bound using option `AllowMissingVariables`) and re-binds again inside the implementation of the language plugin. The second time we bind the program, we don't pass it the option `AllowMissingVariables` and so it fails with `unboud variable` error. I've verified that the issue above don't repro when doing an import for dotnet (probably same for java/yaml) because we use the statically linked function `codegen/{lang}/gen_program.go -> GenerateProgram` ## Solution The problem can be solved by propagating the bind options from the CLI to the language hosts during import so that they know how to bind the program. I've extended the gRPC interface in `GenerateProgramRequest` with a property `Strict` which follows the same logic from `pulumi convert --strict` and made it such that the import command sends `strict=false` to the language plugins when doing `GenerateProgram`. This is consistent with `GenerateProject` that uses the same flag. When `strict=false` we use `pcl.NonStrictBindOptions()` which includes `AllowMissingVariables` . ## Repro Once can test the before and after behaviour by running `pulumi up --yes` on the following TypeScript program: ```ts import * as pulumi from "@pulumi/pulumi"; import * as random from "@pulumi/random"; export class MyComponent extends pulumi.ComponentResource { public readonly randomPetId: pulumi.Output<string>; constructor(name: string, opts?: pulumi.ComponentResourceOptions) { super("example:index:MyComponent", name, {}, opts); const randomPet = new random.RandomPet("randomPet", {}, { parent: this }); this.randomPetId = randomPet.id; this.registerOutputs({ randomPetId: randomPet.id, }); } } const example = new MyComponent("example"); export const randomPetId = example.randomPetId; ``` Then running `pulumi import -f import.json` where `import.json` contains a resource to be imported under the created component (stack=`dev`, project=`importerrors`) ```ts { "nameTable": { "parentComponent": "urn:pulumi:dev::importerrors::example:index:MyComponent::example" }, "resources": [ { "type": "random:index/randomPassword:RandomPassword", "name": "randomPassword", "id": "supersecret", "parent": "parentComponent" } ] } ``` Running this locally I get the following generated code (which previously failed to generate) ```ts import * as pulumi from "@pulumi/pulumi"; import * as random from "@pulumi/random"; const randomPassword = new random.RandomPassword("randomPassword", { length: 11, lower: true, number: true, numeric: true, special: true, upper: true, }, { parent: parentComponent, }); ```
2024-07-25 13:53:44 +00:00
bindOptions := []pcl.BindOption{
pcl.Loader(loader),
[cli/import] Fix undefined variable errors in code generation when imported resources use a parent or provider (#16786) Fixes #15410 Fixes #13339 ## Problem Context When using `pulumi import` we generate code snippets for the resources that were imported. Sometimes the user specifies `--parent parentName=URN` or `--provider providerName=URN` which tweak the parent or provider that the imported resources uses. When using `--parent` or `--provider` the generated code emits a resource option `parent = parentName` (in case of using `--parent`) where `parentName` is an unbound variable. Usually unbound variables would result in a _bind_ error such as `error: undefined variable parentName` when type-checking the program however in the import code generation we specify the bind option `pcl.AllowMissingVariables` which turns that unbound variable errors into warnings and code generation can continue to emit code. This is all good and works as expected. However in the issues linked above, we do get an _error_ for unbound variables in generated code even though we specified `AllowMissingVariables`. The problem as it turns out is when we are trying to generate code via dynamically loaded `LangaugeRuntime` plugins. Specifically for NodeJS and Python, we load `pulumi-language-nodejs` or `pulumi-language-python` and call `GenerateProgram` to get the generated program. That function `GenerateProgram` takes the text _SOURCE_ of the a bound program (one that was bound using option `AllowMissingVariables`) and re-binds again inside the implementation of the language plugin. The second time we bind the program, we don't pass it the option `AllowMissingVariables` and so it fails with `unboud variable` error. I've verified that the issue above don't repro when doing an import for dotnet (probably same for java/yaml) because we use the statically linked function `codegen/{lang}/gen_program.go -> GenerateProgram` ## Solution The problem can be solved by propagating the bind options from the CLI to the language hosts during import so that they know how to bind the program. I've extended the gRPC interface in `GenerateProgramRequest` with a property `Strict` which follows the same logic from `pulumi convert --strict` and made it such that the import command sends `strict=false` to the language plugins when doing `GenerateProgram`. This is consistent with `GenerateProject` that uses the same flag. When `strict=false` we use `pcl.NonStrictBindOptions()` which includes `AllowMissingVariables` . ## Repro Once can test the before and after behaviour by running `pulumi up --yes` on the following TypeScript program: ```ts import * as pulumi from "@pulumi/pulumi"; import * as random from "@pulumi/random"; export class MyComponent extends pulumi.ComponentResource { public readonly randomPetId: pulumi.Output<string>; constructor(name: string, opts?: pulumi.ComponentResourceOptions) { super("example:index:MyComponent", name, {}, opts); const randomPet = new random.RandomPet("randomPet", {}, { parent: this }); this.randomPetId = randomPet.id; this.registerOutputs({ randomPetId: randomPet.id, }); } } const example = new MyComponent("example"); export const randomPetId = example.randomPetId; ``` Then running `pulumi import -f import.json` where `import.json` contains a resource to be imported under the created component (stack=`dev`, project=`importerrors`) ```ts { "nameTable": { "parentComponent": "urn:pulumi:dev::importerrors::example:index:MyComponent::example" }, "resources": [ { "type": "random:index/randomPassword:RandomPassword", "name": "randomPassword", "id": "supersecret", "parent": "parentComponent" } ] } ``` Running this locally I get the following generated code (which previously failed to generate) ```ts import * as pulumi from "@pulumi/pulumi"; import * as random from "@pulumi/random"; const randomPassword = new random.RandomPassword("randomPassword", { length: 11, lower: true, number: true, numeric: true, special: true, upper: true, }, { parent: parentComponent, }); ```
2024-07-25 13:53:44 +00:00
// for nodejs, prefer output-versioned invokes
pcl.PreferOutputVersionedInvokes,
}
if !req.Strict {
bindOptions = append(bindOptions, pcl.NonStrictBindOptions()...)
}
program, diags, err := pcl.BindProgram(parser.Files, bindOptions...)
2023-05-12 13:38:36 +00:00
if err != nil {
return nil, err
}
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
if diags.HasErrors() {
return &pulumirpc.GenerateProgramResponse{
Diagnostics: rpcDiagnostics,
}, nil
}
if program == nil {
return nil, errors.New("internal error program was nil")
2023-05-12 13:38:36 +00:00
}
files, diags, err := codegen.GenerateProgram(program)
if err != nil {
return nil, err
}
rpcDiagnostics = append(rpcDiagnostics, plugin.HclDiagnosticsToRPCDiagnostics(diags)...)
2023-05-12 13:38:36 +00:00
return &pulumirpc.GenerateProgramResponse{
Source: files,
Diagnostics: rpcDiagnostics,
}, nil
}
func (host *nodeLanguageHost) GeneratePackage(
ctx context.Context, req *pulumirpc.GeneratePackageRequest,
) (*pulumirpc.GeneratePackageResponse, error) {
loader, err := schema.NewLoaderClient(req.LoaderTarget)
2023-05-12 13:38:36 +00:00
if err != nil {
return nil, err
}
var spec schema.PackageSpec
err = json.Unmarshal([]byte(req.Schema), &spec)
if err != nil {
return nil, err
}
pkg, diags, err := schema.BindSpec(spec, loader)
if err != nil {
return nil, err
}
Return diagnostics from GeneratePackage (#14661) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/14660. Fairly simple change to bring the GeneratePackage RPC method into alignment with the other codegen methods and use returned diagnostics rather than just error values. `gen-sdk` is updated to print those diagnostics, and the python/node/go runtimes updated to return the diagnostics from schema binding as diagnostics rather than just an error value. Might be worth at some point seeing if the rest of package generation could use diagnostics rather than error values, but that's a larger lift. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-05 17:47:52 +00:00
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
2023-05-12 13:38:36 +00:00
if diags.HasErrors() {
Return diagnostics from GeneratePackage (#14661) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/14660. Fairly simple change to bring the GeneratePackage RPC method into alignment with the other codegen methods and use returned diagnostics rather than just error values. `gen-sdk` is updated to print those diagnostics, and the python/node/go runtimes updated to return the diagnostics from schema binding as diagnostics rather than just an error value. Might be worth at some point seeing if the rest of package generation could use diagnostics rather than error values, but that's a larger lift. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-05 17:47:52 +00:00
return &pulumirpc.GeneratePackageResponse{
Diagnostics: rpcDiagnostics,
}, nil
2023-05-12 13:38:36 +00:00
}
Use the local artifacts, not released artifacts in conformance tests (#15777) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This threads the "local_dependencies" property through to GeneratePackage, following exactly the same semantics as for "GenerateProgram". Fixes https://github.com/pulumi/pulumi/issues/15074. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-03-26 13:10:34 +00:00
files, err := codegen.GeneratePackage("pulumi-language-nodejs", pkg, req.ExtraFiles, req.LocalDependencies)
2023-05-12 13:38:36 +00:00
if err != nil {
return nil, err
}
for filename, data := range files {
outPath := filepath.Join(req.Directory, filename)
2023-05-12 13:38:36 +00:00
err := os.MkdirAll(filepath.Dir(outPath), 0o700)
if err != nil {
return nil, fmt.Errorf("could not create output directory %s: %w", filepath.Dir(filename), err)
}
err = os.WriteFile(outPath, data, 0o600)
if err != nil {
return nil, fmt.Errorf("could not write output file %s: %w", filename, err)
}
}
Return diagnostics from GeneratePackage (#14661) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes https://github.com/pulumi/pulumi/issues/14660. Fairly simple change to bring the GeneratePackage RPC method into alignment with the other codegen methods and use returned diagnostics rather than just error values. `gen-sdk` is updated to print those diagnostics, and the python/node/go runtimes updated to return the diagnostics from schema binding as diagnostics rather than just an error value. Might be worth at some point seeing if the rest of package generation could use diagnostics rather than error values, but that's a larger lift. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-05 17:47:52 +00:00
return &pulumirpc.GeneratePackageResponse{
Diagnostics: rpcDiagnostics,
}, nil
2023-05-12 13:38:36 +00:00
}
func readPackageJSON(packageJSONPath string) (map[string]interface{}, error) {
packageJSONData, err := os.ReadFile(packageJSONPath)
if err != nil {
return nil, fmt.Errorf("read package.json: %w", err)
}
var packageJSON map[string]interface{}
err = json.Unmarshal(packageJSONData, &packageJSON)
if err != nil {
return nil, fmt.Errorf("unmarshal package.json: %w", err)
}
return packageJSON, nil
}
func (host *nodeLanguageHost) Pack(ctx context.Context, req *pulumirpc.PackRequest) (*pulumirpc.PackResponse, error) {
// Verify npm exists and is set up: npm, user login
npm, err := executable.FindExecutable("npm")
if err != nil {
return nil, fmt.Errorf("find npm: %w", err)
}
// Annoyingly the engine will call Pack for the core SDK which is not setup in at all the same way as the
// generated sdks, so we have to detect that and do a big branch to pack it totally differently.
packageJSON, err := readPackageJSON(filepath.Join(req.PackageDirectory, "package.json"))
if err != nil {
return nil, err
}
// TODO: We're just going to write to stderr for now, but we should probably have a way to return this
// directly to the engine hosting us. That's a bit awkward because we need the subprocesses to write to
// this as well, and the host side of this pipe might be a tty and if it is we want the subprocesses to
// think they're connected to a tty as well. We've solved this problem before in two slightly different
// ways for InstallDependencies and RunPlugin, it would be good to come up with a clean way to do this for
// all these cases.
writeString := func(s string) error {
_, err = os.Stderr.Write([]byte(s))
return err
}
if packageJSON["name"] == "@pulumi/pulumi" {
// This is pretty much a copy of the makefiles build_package. Short term we should see about changing
// the makefile to just build the nodejs plugin first and then simply invoke "pulumi package
// pack-sdk". Long term we should try and unify the style of the code sdk with that of generated sdks
// so we don't need this special case.
yarn, err := executable.FindExecutable("yarn")
if err != nil {
return nil, fmt.Errorf("find yarn: %w", err)
}
err = writeString("$ yarn install --frozen-lockfile\n")
if err != nil {
return nil, fmt.Errorf("write to output: %w", err)
}
yarnInstallCmd := exec.Command(yarn, "install", "--frozen-lockfile")
yarnInstallCmd.Dir = req.PackageDirectory
if err := runWithOutput(yarnInstallCmd, os.Stdout, os.Stderr); err != nil {
return nil, fmt.Errorf("yarn install: %w", err)
}
err = writeString("$ yarn run tsc\n")
if err != nil {
return nil, fmt.Errorf("write to output: %w", err)
}
yarnTscCmd := exec.Command(yarn, "run", "tsc")
yarnTscCmd.Dir = req.PackageDirectory
if err := runWithOutput(yarnTscCmd, os.Stdout, os.Stderr); err != nil {
return nil, fmt.Errorf("yarn run tsc: %w", err)
}
Add matrix testing (#13705) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Adds the first pass of matrix testing. Matrix testing allows us to define tests once in pulumi/pulumi via PCL and then run those tests against each language plugin to verify code generation and runtime correctness. Rather than packing matrix tests and all the associated data and machinery into the CLI itself we define a new Go package at cmd/pulumi-test-lanaguage. This depends on pkg and runs the deployment engine in a unique way for matrix tests but it is running the proper deployment engine with a proper backend (always filestate, using $TEMP). Currently only NodeJS is hooked up to run these tests, and all the code for that currently lives in sdk/nodejs/cmd/pulumi-language-nodejs/language_test.go. I expect we'll move that helper code to sdk/go/common and use it in each language plugin to run the tests in the same way. This first pass includes 3 simple tests: * l1-empty that runs an empty PCL file and checks just a stack is created * l1-output-bool that runs a PCL program that returns two stack outputs of `true` and `false * l2-resource-simple that runs a PCL program creating a simple resource with a single bool property These tests are themselves tested with a mock language runtime. This verifies the behavior of the matrix test framework for both correct and incorrect language hosts (that is some the mock language runtimes purposefully cause errors or compute the wrong result). There are a number of things missing from from the core framework still, but I feel don't block getting this first pass merged and starting to be used. 1. The tests can not currently run in parallel. That is calling RunLanguageTest in parallel will break things. This is due to two separate problems. Firstly is that the SDK snapshot's are not safe to write in parallel (when PULUMI_ACCEPT is true), this should be fairly easy to fix by doing a write to dst-{random} and them atomic move to dst. Secondly is that the deployment engine itself has mutable global state, short term we should probably just lock around that part RunLanguageTest, long term it would be good to clean that up. 2. We need a way to verify "preview" behavior, I think this is probably just a variation of the tests that would call `stack.Preview` and not pass a snapshot to `assert`. 3. stdout, stderr and log messages are returned in bulk at the end of the test. Plus there are a couple of calls to the language runtime that don't correctly thread stdout/stderr to use and so default to the process `os.Stdout/Stderr`. stdout/stderr streaming shows up in a load of other places as well so I'm thinking of a clean way to handle all of them together. Log message streaming we can probably do by just turning RunLanguageTest to a streaming grpc call. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. --> --------- Co-authored-by: Abhinav Gupta <abhinav@pulumi.com>
2023-09-13 15:17:46 +00:00
Vendor TypeScript and ts-node (#15622) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description Fixes https://github.com/pulumi/pulumi/issues/15733 Historically these packages were direct dependencies of `@pulumi/pulumi`. To decouple the node SDK from the precise version of TypeScript, the packages are now declared as optional peer pependencies of `@pulumi/pulumi` and customers can pick the versions they want. The reason we mark the peer dependencies as *optional* is to prevent package managers from automatically installing them. This avoids the situation where the package manger would install a more recent version of TypeScript without the user explictly opting in. Newer versions have stricter type checks, and can thus stop existing programs from running successfully. When the peer dependencies are not present, we load the vendored versions of the modules. ## Checklist - [ ] I have run `make tidy` to update any new dependencies - [ ] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-10 15:26:37 +00:00
// "tsc" doesn't copy in the "proto" and "vendor" directories.
Add matrix testing (#13705) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Adds the first pass of matrix testing. Matrix testing allows us to define tests once in pulumi/pulumi via PCL and then run those tests against each language plugin to verify code generation and runtime correctness. Rather than packing matrix tests and all the associated data and machinery into the CLI itself we define a new Go package at cmd/pulumi-test-lanaguage. This depends on pkg and runs the deployment engine in a unique way for matrix tests but it is running the proper deployment engine with a proper backend (always filestate, using $TEMP). Currently only NodeJS is hooked up to run these tests, and all the code for that currently lives in sdk/nodejs/cmd/pulumi-language-nodejs/language_test.go. I expect we'll move that helper code to sdk/go/common and use it in each language plugin to run the tests in the same way. This first pass includes 3 simple tests: * l1-empty that runs an empty PCL file and checks just a stack is created * l1-output-bool that runs a PCL program that returns two stack outputs of `true` and `false * l2-resource-simple that runs a PCL program creating a simple resource with a single bool property These tests are themselves tested with a mock language runtime. This verifies the behavior of the matrix test framework for both correct and incorrect language hosts (that is some the mock language runtimes purposefully cause errors or compute the wrong result). There are a number of things missing from from the core framework still, but I feel don't block getting this first pass merged and starting to be used. 1. The tests can not currently run in parallel. That is calling RunLanguageTest in parallel will break things. This is due to two separate problems. Firstly is that the SDK snapshot's are not safe to write in parallel (when PULUMI_ACCEPT is true), this should be fairly easy to fix by doing a write to dst-{random} and them atomic move to dst. Secondly is that the deployment engine itself has mutable global state, short term we should probably just lock around that part RunLanguageTest, long term it would be good to clean that up. 2. We need a way to verify "preview" behavior, I think this is probably just a variation of the tests that would call `stack.Preview` and not pass a snapshot to `assert`. 3. stdout, stderr and log messages are returned in bulk at the end of the test. Plus there are a couple of calls to the language runtime that don't correctly thread stdout/stderr to use and so default to the process `os.Stdout/Stderr`. stdout/stderr streaming shows up in a load of other places as well so I'm thinking of a clean way to handle all of them together. Log message streaming we can probably do by just turning RunLanguageTest to a streaming grpc call. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. --> --------- Co-authored-by: Abhinav Gupta <abhinav@pulumi.com>
2023-09-13 15:17:46 +00:00
err = fsutil.CopyFile(
filepath.Join(req.PackageDirectory, "bin", "proto"),
filepath.Join(req.PackageDirectory, "proto"),
nil)
if err != nil {
return nil, fmt.Errorf("copy proto: %w", err)
}
Vendor TypeScript and ts-node (#15622) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description Fixes https://github.com/pulumi/pulumi/issues/15733 Historically these packages were direct dependencies of `@pulumi/pulumi`. To decouple the node SDK from the precise version of TypeScript, the packages are now declared as optional peer pependencies of `@pulumi/pulumi` and customers can pick the versions they want. The reason we mark the peer dependencies as *optional* is to prevent package managers from automatically installing them. This avoids the situation where the package manger would install a more recent version of TypeScript without the user explictly opting in. Newer versions have stricter type checks, and can thus stop existing programs from running successfully. When the peer dependencies are not present, we load the vendored versions of the modules. ## Checklist - [ ] I have run `make tidy` to update any new dependencies - [ ] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-04-10 15:26:37 +00:00
err = fsutil.CopyFile(
filepath.Join(req.PackageDirectory, "bin", "vendor"),
filepath.Join(req.PackageDirectory, "vendor"),
nil)
if err != nil {
return nil, fmt.Errorf("copy vendor: %w", err)
}
Add matrix testing (#13705) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Adds the first pass of matrix testing. Matrix testing allows us to define tests once in pulumi/pulumi via PCL and then run those tests against each language plugin to verify code generation and runtime correctness. Rather than packing matrix tests and all the associated data and machinery into the CLI itself we define a new Go package at cmd/pulumi-test-lanaguage. This depends on pkg and runs the deployment engine in a unique way for matrix tests but it is running the proper deployment engine with a proper backend (always filestate, using $TEMP). Currently only NodeJS is hooked up to run these tests, and all the code for that currently lives in sdk/nodejs/cmd/pulumi-language-nodejs/language_test.go. I expect we'll move that helper code to sdk/go/common and use it in each language plugin to run the tests in the same way. This first pass includes 3 simple tests: * l1-empty that runs an empty PCL file and checks just a stack is created * l1-output-bool that runs a PCL program that returns two stack outputs of `true` and `false * l2-resource-simple that runs a PCL program creating a simple resource with a single bool property These tests are themselves tested with a mock language runtime. This verifies the behavior of the matrix test framework for both correct and incorrect language hosts (that is some the mock language runtimes purposefully cause errors or compute the wrong result). There are a number of things missing from from the core framework still, but I feel don't block getting this first pass merged and starting to be used. 1. The tests can not currently run in parallel. That is calling RunLanguageTest in parallel will break things. This is due to two separate problems. Firstly is that the SDK snapshot's are not safe to write in parallel (when PULUMI_ACCEPT is true), this should be fairly easy to fix by doing a write to dst-{random} and them atomic move to dst. Secondly is that the deployment engine itself has mutable global state, short term we should probably just lock around that part RunLanguageTest, long term it would be good to clean that up. 2. We need a way to verify "preview" behavior, I think this is probably just a variation of the tests that would call `stack.Preview` and not pass a snapshot to `assert`. 3. stdout, stderr and log messages are returned in bulk at the end of the test. Plus there are a couple of calls to the language runtime that don't correctly thread stdout/stderr to use and so default to the process `os.Stdout/Stderr`. stdout/stderr streaming shows up in a load of other places as well so I'm thinking of a clean way to handle all of them together. Log message streaming we can probably do by just turning RunLanguageTest to a streaming grpc call. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. --> --------- Co-authored-by: Abhinav Gupta <abhinav@pulumi.com>
2023-09-13 15:17:46 +00:00
} else {
// Before we can build the package we need to install it's dependencies.
err = writeString("$ npm install\n")
if err != nil {
return nil, fmt.Errorf("write to output: %w", err)
}
npmInstallCmd := exec.Command(npm, "install")
npmInstallCmd.Dir = req.PackageDirectory
if err := runWithOutput(npmInstallCmd, os.Stdout, os.Stderr); err != nil {
return nil, fmt.Errorf("npm install: %w", err)
}
// Pulumi SDKs always define a build command that will run tsc writing to a bin directory.
// So we can run that, then edit the package.json in that directory, and then pack it.
err = writeString("$ npm run build\n")
if err != nil {
return nil, fmt.Errorf("write to output: %w", err)
}
npmBuildCmd := exec.Command(npm, "run", "build")
npmBuildCmd.Dir = req.PackageDirectory
if err := runWithOutput(npmBuildCmd, os.Stdout, os.Stderr); err != nil {
return nil, fmt.Errorf("npm run build: %w", err)
}
// "build" in SDKs isn't setup to copy the package.json to ./bin/
err = fsutil.CopyFile(
filepath.Join(req.PackageDirectory, "bin", "package.json"),
filepath.Join(req.PackageDirectory, "package.json"),
nil)
if err != nil {
return nil, fmt.Errorf("copy package.json: %w", err)
}
}
err = writeString("$ npm pack\n")
if err != nil {
return nil, fmt.Errorf("write to output: %w", err)
}
var stdoutBuffer bytes.Buffer
npmPackCmd := exec.Command(npm,
"pack",
filepath.Join(req.PackageDirectory, "bin"),
"--pack-destination", req.DestinationDirectory)
npmPackCmd.Stdout = &stdoutBuffer
npmPackCmd.Stderr = struct{ io.Writer }{os.Stderr}
err = npmPackCmd.Run()
if err != nil {
return nil, fmt.Errorf("npm pack: %w", err)
}
artifactName := strings.TrimSpace(stdoutBuffer.String())
return &pulumirpc.PackResponse{
ArtifactPath: filepath.Join(req.DestinationDirectory, artifactName),
}, nil
}
// Nodejs sometimes sets stdout/stderr to non-blocking mode. When a nodejs subprocess is directly
// handed the go process's stdout/stderr file descriptors, nodejs's non-blocking configuration goes
// unnoticed by go, and a write from go can result in an error `write /dev/stdout: resource
// temporarily unavailable`.
//
// The solution to this is to not provide nodejs with the go process's stdout/stderr file
// descriptors, and instead proxy the writes through something else.
// In https://github.com/pulumi/pulumi/pull/16504 we used Cmd.StdoutPipe/StderrPipe for this.
// However this introduced a potential bug, as it is not safe to use these specific pipes along
// with Cmd.Run. The issue is that these pipes will be closed as soon as the process exits, which
// can lead to missing data when stdout/stderr are slow, or worse an error due to an attempted read
// from a closed pipe. (Creating and closing our own os.Pipes for this would be fine.)
//
// A simpler workaround is to wrap stdout/stderr in an io.Writer. As with the pipes, this makes
// exec.Cmd use a copying goroutine to shuffle the data from the subprocess to stdout/stderr.
//
// Cmd.Run will wait for all the data to be copied before returning, ensuring we do not miss any data.
//
// Non-blocking issue: https://github.com/golang/go/issues/58408#issuecomment-1423621323
// StdoutPipe/StderrPipe issues: https://pkg.go.dev/os/exec#Cmd.StdoutPipe
// Waiting for data: https://cs.opensource.google/go/go/+/refs/tags/go1.22.5:src/os/exec/exec.go;l=201
func runWithOutput(cmd *exec.Cmd, stdout, stderr io.Writer) error {
cmd.Stdout = struct{ io.Writer }{stdout}
cmd.Stderr = struct{ io.Writer }{stderr}
return cmd.Run()
}
// oomSniffer is a scanner that detects OOM errors in the output of a nodejs process.
type oomSniffer struct {
detected bool
timeout time.Duration
waitChan chan struct{}
}
func newOOMSniffer() *oomSniffer {
return &oomSniffer{
timeout: 15 * time.Second,
waitChan: make(chan struct{}),
}
}
func (o *oomSniffer) Detected() bool {
return o.detected
}
// Wait waits for the OOM sniffer to either:
// - detect an OOM error
// - the timeout to expire
// - the reader to be closed
//
// Call Wait to ensure we've read all the output from the scanned process after it exits.
func (o *oomSniffer) Wait() {
select {
case <-o.waitChan:
case <-time.After(o.timeout):
}
}
func (o *oomSniffer) Message() string {
return "Detected a possible out of memory error. Consider increasing the memory available to the nodejs process " +
"by setting the `nodeargs` runtime option in Pulumi.yaml to `nodeargs: --max-old-space-size=<size>` where " +
"`<size>` is the maximum memory in megabytes that can be allocated to nodejs. " +
"See https://www.pulumi.com/docs/concepts/projects/project-file/#runtime-options"
}
func (o *oomSniffer) Scan(r io.Reader) {
scanner := bufio.NewScanner(r)
go func() {
for scanner.Scan() {
line := scanner.Text()
if !o.detected && strings.Contains(line, "<--- Last few GCs --->") /* "Normal" OOM output */ ||
// Because we hook into the debugger API, the OOM error message can be obscured by
// a failed assertion in the debugger https://github.com/pulumi/pulumi/issues/16596.
strings.Contains(line, "Check failed: needs_context && current_scope_ = closure_scope_") {
o.detected = true
close(o.waitChan)
}
}
contract.IgnoreError(scanner.Err())
if !o.detected {
close(o.waitChan)
}
}()
}