2022-09-12 21:42:27 +00:00
|
|
|
// Copyright 2016-2022, Pulumi Corporation.
|
2018-05-22 19:43:36 +00:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
2018-02-10 02:15:04 +00:00
|
|
|
|
2018-02-12 03:22:23 +00:00
|
|
|
// pulumi-language-nodejs serves as the "language host" for Pulumi
|
2018-02-10 02:15:04 +00:00
|
|
|
// programs written in NodeJS. It is ultimately responsible for spawning the
|
|
|
|
// language runtime that executes the program.
|
|
|
|
//
|
|
|
|
// The program being executed is executed by a shim script called
|
2018-02-12 03:22:23 +00:00
|
|
|
// `pulumi-language-nodejs-exec`. This script is written in the hosted
|
2018-02-10 02:15:04 +00:00
|
|
|
// language (in this case, node) and is responsible for initiating RPC
|
|
|
|
// links to the resource monitor and engine.
|
|
|
|
//
|
|
|
|
// It's therefore the responsibility of this program to implement
|
|
|
|
// the LanguageHostServer endpoint by spawning instances of
|
2018-02-12 03:22:23 +00:00
|
|
|
// `pulumi-language-nodejs-exec` and forwarding the RPC request arguments
|
2018-02-10 02:15:04 +00:00
|
|
|
// to the command-line.
|
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2024-07-23 16:10:43 +00:00
|
|
|
"bufio"
|
2023-07-27 21:39:36 +00:00
|
|
|
"bytes"
|
2018-02-10 02:15:04 +00:00
|
|
|
"context"
|
|
|
|
"encoding/json"
|
2022-11-04 10:20:29 +00:00
|
|
|
"errors"
|
2018-02-10 02:15:04 +00:00
|
|
|
"flag"
|
|
|
|
"fmt"
|
2024-06-27 18:34:31 +00:00
|
|
|
"io"
|
2023-01-23 17:18:14 +00:00
|
|
|
"io/fs"
|
2018-02-10 02:15:04 +00:00
|
|
|
"os"
|
|
|
|
"os/exec"
|
2023-08-29 15:42:31 +00:00
|
|
|
"os/signal"
|
2018-02-12 03:22:23 +00:00
|
|
|
"path/filepath"
|
2023-12-12 12:19:42 +00:00
|
|
|
"strconv"
|
2018-02-10 02:15:04 +00:00
|
|
|
"strings"
|
2022-06-06 12:28:00 +00:00
|
|
|
"time"
|
2018-02-10 02:15:04 +00:00
|
|
|
|
2022-01-05 02:54:38 +00:00
|
|
|
"github.com/blang/semver"
|
|
|
|
"github.com/google/shlex"
|
|
|
|
"github.com/hashicorp/go-multierror"
|
2019-10-15 05:08:06 +00:00
|
|
|
opentracing "github.com/opentracing/opentracing-go"
|
2022-01-05 02:54:38 +00:00
|
|
|
"google.golang.org/grpc"
|
2023-01-11 19:54:31 +00:00
|
|
|
"google.golang.org/grpc/credentials/insecure"
|
2024-01-17 09:35:20 +00:00
|
|
|
"google.golang.org/protobuf/types/known/emptypb"
|
2022-01-05 02:54:38 +00:00
|
|
|
|
2021-03-17 13:20:05 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
|
2022-08-15 13:55:04 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/executable"
|
2023-07-27 21:39:36 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/fsutil"
|
2021-03-17 13:20:05 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/version"
|
2023-05-12 13:38:36 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
|
2022-04-03 14:54:59 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/nodejs/npm"
|
2021-03-17 13:20:05 +00:00
|
|
|
pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
|
2023-05-12 13:38:36 +00:00
|
|
|
|
|
|
|
hclsyntax "github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax"
|
|
|
|
codegen "github.com/pulumi/pulumi/pkg/v3/codegen/nodejs"
|
|
|
|
"github.com/pulumi/pulumi/pkg/v3/codegen/pcl"
|
|
|
|
"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
|
2018-02-10 02:15:04 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
2018-11-05 21:36:35 +00:00
|
|
|
// The path to the "run" program which will spawn the rest of the language host. This may be overridden with
|
2018-04-30 23:10:01 +00:00
|
|
|
// PULUMI_LANGUAGE_NODEJS_RUN_PATH, which we do in some testing cases.
|
2018-09-06 00:04:51 +00:00
|
|
|
defaultRunPath = "@pulumi/pulumi/cmd/run"
|
2018-02-10 02:15:04 +00:00
|
|
|
|
|
|
|
// The runtime expects the config object to be saved to this environment variable.
|
|
|
|
pulumiConfigVar = "PULUMI_CONFIG"
|
2019-03-20 18:54:32 +00:00
|
|
|
|
2021-05-18 16:48:08 +00:00
|
|
|
// The runtime expects the array of secret config keys to be saved to this environment variable.
|
2023-01-06 00:07:45 +00:00
|
|
|
//nolint:gosec
|
2021-05-18 16:48:08 +00:00
|
|
|
pulumiConfigSecretKeysVar = "PULUMI_CONFIG_SECRET_KEYS"
|
|
|
|
|
2019-03-20 18:54:32 +00:00
|
|
|
// A exit-code we recognize when the nodejs process exits. If we see this error, there's no
|
|
|
|
// need for us to print any additional error messages since the user already got a a good
|
|
|
|
// one they can handle.
|
|
|
|
nodeJSProcessExitedAfterShowingUserActionableMessage = 32
|
2018-02-10 02:15:04 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// Launches the language host RPC endpoint, which in turn fires
|
|
|
|
// up an RPC server implementing the LanguageRuntimeServer RPC
|
|
|
|
// endpoint.
|
|
|
|
func main() {
|
|
|
|
var tracing string
|
|
|
|
flag.StringVar(&tracing, "tracing", "",
|
|
|
|
"Emit tracing to a Zipkin-compatible tracing endpoint")
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
flag.Bool("typescript", true,
|
|
|
|
"[obsolete] Use ts-node at runtime to support typescript source natively")
|
|
|
|
flag.String("root", "", "[obsolete] Project root path to use")
|
|
|
|
flag.String("tsconfig", "",
|
|
|
|
"[obsolete] Path to tsconfig.json to use")
|
|
|
|
flag.String("nodeargs", "", "[obsolete] Arguments for the Node process")
|
2024-06-21 11:35:06 +00:00
|
|
|
flag.String("packagemanager", "", "[obsolete] Packagemanager to use (auto, npm, yarn or pnpm)")
|
2018-02-10 02:15:04 +00:00
|
|
|
flag.Parse()
|
2018-06-25 05:47:54 +00:00
|
|
|
|
2018-02-10 02:15:04 +00:00
|
|
|
args := flag.Args()
|
2018-05-15 22:28:00 +00:00
|
|
|
logging.InitLogging(false, 0, false)
|
2019-06-18 21:39:42 +00:00
|
|
|
cmdutil.InitTracing("pulumi-language-nodejs", "pulumi-language-nodejs", tracing)
|
2018-02-10 02:15:04 +00:00
|
|
|
|
|
|
|
// Optionally pluck out the engine so we can do logging, etc.
|
|
|
|
var engineAddress string
|
2018-02-12 03:22:23 +00:00
|
|
|
if len(args) > 0 {
|
|
|
|
engineAddress = args[0]
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
|
|
|
|
2023-08-29 15:42:31 +00:00
|
|
|
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
|
2022-06-06 12:28:00 +00:00
|
|
|
// map the context Done channel to the rpcutil boolean cancel channel
|
|
|
|
cancelChannel := make(chan bool)
|
|
|
|
go func() {
|
|
|
|
<-ctx.Done()
|
2023-08-29 15:42:31 +00:00
|
|
|
cancel() // deregister the interrupt handler
|
2022-06-06 12:28:00 +00:00
|
|
|
close(cancelChannel)
|
|
|
|
}()
|
|
|
|
err := rpcutil.Healthcheck(ctx, engineAddress, 5*time.Minute, cancel)
|
|
|
|
if err != nil {
|
2022-11-04 10:20:29 +00:00
|
|
|
cmdutil.Exit(fmt.Errorf("could not start health check host RPC server: %w", err))
|
2022-06-06 12:28:00 +00:00
|
|
|
}
|
|
|
|
|
2018-02-10 02:15:04 +00:00
|
|
|
// Fire up a gRPC server, letting the kernel choose a free port.
|
2022-11-01 15:15:09 +00:00
|
|
|
handle, err := rpcutil.ServeWithOptions(rpcutil.ServeOptions{
|
|
|
|
Cancel: cancelChannel,
|
|
|
|
Init: func(srv *grpc.Server) error {
|
2024-03-28 15:44:25 +00:00
|
|
|
host := newLanguageHost(engineAddress, tracing, false /* forceTsc */)
|
2018-02-10 02:15:04 +00:00
|
|
|
pulumirpc.RegisterLanguageRuntimeServer(srv, host)
|
|
|
|
return nil
|
|
|
|
},
|
2022-11-01 15:15:09 +00:00
|
|
|
Options: rpcutil.OpenTracingServerInterceptorOptions(nil),
|
|
|
|
})
|
2018-02-10 02:15:04 +00:00
|
|
|
if err != nil {
|
2022-11-04 10:20:29 +00:00
|
|
|
cmdutil.Exit(fmt.Errorf("could not start language host RPC server: %w", err))
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Otherwise, print out the port so that the spawner knows how to reach us.
|
2022-11-01 15:15:09 +00:00
|
|
|
fmt.Printf("%d\n", handle.Port)
|
2018-02-12 03:22:23 +00:00
|
|
|
|
2018-02-10 02:15:04 +00:00
|
|
|
// And finally wait for the server to stop serving.
|
2022-11-01 15:15:09 +00:00
|
|
|
if err := <-handle.Done; err != nil {
|
2022-11-04 10:20:29 +00:00
|
|
|
cmdutil.Exit(fmt.Errorf("language host RPC stopped serving: %w", err))
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
|
|
|
}
|
2018-09-06 00:04:51 +00:00
|
|
|
|
|
|
|
// locateModule resolves a node module name to a file path that can be loaded
|
2023-08-02 14:17:43 +00:00
|
|
|
func locateModule(ctx context.Context, mod, programDir, nodeBin string) (string, error) {
|
2024-07-12 15:31:43 +00:00
|
|
|
script := fmt.Sprintf(`try {
|
2024-05-10 13:34:05 +00:00
|
|
|
console.log(require.resolve('%s'));
|
|
|
|
} catch (error) {
|
|
|
|
if (error.code === 'MODULE_NOT_FOUND') {
|
|
|
|
console.error("It looks like the Pulumi SDK has not been installed. Have you run pulumi install?")
|
|
|
|
} else {
|
|
|
|
console.error(error.message);
|
|
|
|
}
|
|
|
|
process.exit(1);
|
2024-07-12 15:31:43 +00:00
|
|
|
}`, mod)
|
|
|
|
// The Volta package manager installs shim executables that route to the user's chosen nodejs
|
|
|
|
// version. On Windows this does not properly handle arguments with newlines, so we need to
|
|
|
|
// ensure that the script is a single line.
|
|
|
|
// https://github.com/pulumi/pulumi/issues/16393
|
|
|
|
script = strings.Replace(script, "\n", "", -1)
|
|
|
|
args := []string{"-e", script}
|
2022-07-19 00:27:15 +00:00
|
|
|
|
|
|
|
tracingSpan, _ := opentracing.StartSpanFromContext(ctx,
|
|
|
|
"locateModule",
|
|
|
|
opentracing.Tag{Key: "module", Value: mod},
|
|
|
|
opentracing.Tag{Key: "component", Value: "exec.Command"},
|
|
|
|
opentracing.Tag{Key: "command", Value: nodeBin},
|
|
|
|
opentracing.Tag{Key: "args", Value: args})
|
|
|
|
|
|
|
|
defer tracingSpan.Finish()
|
|
|
|
|
|
|
|
cmd := exec.Command(nodeBin, args...)
|
2023-08-02 14:17:43 +00:00
|
|
|
cmd.Dir = programDir
|
2018-09-06 00:04:51 +00:00
|
|
|
out, err := cmd.Output()
|
|
|
|
if err != nil {
|
2024-05-10 13:34:05 +00:00
|
|
|
if ee, ok := err.(*exec.ExitError); ok {
|
|
|
|
return "", errors.New(string(ee.Stderr))
|
|
|
|
}
|
2018-09-06 00:04:51 +00:00
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return strings.TrimSpace(string(out)), nil
|
|
|
|
}
|
2018-02-10 02:15:04 +00:00
|
|
|
|
|
|
|
// nodeLanguageHost implements the LanguageRuntimeServer interface
|
|
|
|
// for use as an API endpoint.
|
|
|
|
type nodeLanguageHost struct {
|
2022-12-14 19:20:26 +00:00
|
|
|
pulumirpc.UnimplementedLanguageRuntimeServer
|
|
|
|
|
2018-02-12 03:22:23 +00:00
|
|
|
engineAddress string
|
|
|
|
tracing string
|
2024-03-28 15:44:25 +00:00
|
|
|
|
|
|
|
// used by language conformance tests to force TSC usage
|
|
|
|
forceTsc bool
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type nodeOptions struct {
|
|
|
|
// Use ts-node at runtime to support typescript source natively
|
|
|
|
typescript bool
|
|
|
|
// Path to tsconfig.json to use
|
|
|
|
tsconfigpath string
|
|
|
|
// Arguments for the Node process
|
|
|
|
nodeargs string
|
2024-06-21 11:35:06 +00:00
|
|
|
// The packagemanger to use to install dependencies.
|
|
|
|
// One of auto, npm, yarn or pnpm, defaults to auto.
|
|
|
|
packagemanager npm.PackageManagerType
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
}
|
|
|
|
|
2024-06-21 11:35:06 +00:00
|
|
|
func parseOptions(options map[string]interface{}) (nodeOptions, error) {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
// typescript defaults to true
|
|
|
|
nodeOptions := nodeOptions{
|
|
|
|
typescript: true,
|
|
|
|
}
|
|
|
|
|
|
|
|
if typescript, ok := options["typescript"]; ok {
|
|
|
|
if ts, ok := typescript.(bool); ok {
|
|
|
|
nodeOptions.typescript = ts
|
|
|
|
} else {
|
|
|
|
return nodeOptions, errors.New("typescript option must be a boolean")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if tsconfigpath, ok := options["tsconfig"]; ok {
|
|
|
|
if tsconfig, ok := tsconfigpath.(string); ok {
|
|
|
|
nodeOptions.tsconfigpath = tsconfig
|
|
|
|
} else {
|
|
|
|
return nodeOptions, errors.New("tsconfigpath option must be a string")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if nodeargs, ok := options["nodeargs"]; ok {
|
|
|
|
if args, ok := nodeargs.(string); ok {
|
|
|
|
nodeOptions.nodeargs = args
|
|
|
|
} else {
|
|
|
|
return nodeOptions, errors.New("nodeargs option must be a string")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-06-21 11:35:06 +00:00
|
|
|
if packagemanager, ok := options["packagemanager"]; ok {
|
|
|
|
if pm, ok := packagemanager.(string); ok {
|
|
|
|
switch pm {
|
|
|
|
case "auto":
|
|
|
|
nodeOptions.packagemanager = npm.AutoPackageManager
|
|
|
|
case "npm":
|
|
|
|
nodeOptions.packagemanager = npm.NpmPackageManager
|
|
|
|
case "yarn":
|
|
|
|
nodeOptions.packagemanager = npm.YarnPackageManager
|
|
|
|
case "pnpm":
|
|
|
|
nodeOptions.packagemanager = npm.PnpmPackageManager
|
|
|
|
default:
|
|
|
|
return nodeOptions, fmt.Errorf("packagemanager option must be one of auto, npm, yarn or pnpm, got %q", pm)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
return nodeOptions, errors.New("packagemanager option must be a string")
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
nodeOptions.packagemanager = npm.AutoPackageManager
|
|
|
|
}
|
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
return nodeOptions, nil
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
|
|
|
|
2022-04-03 14:54:59 +00:00
|
|
|
func newLanguageHost(
|
2024-03-28 15:44:25 +00:00
|
|
|
engineAddress, tracing string, forceTsc bool,
|
2023-03-03 16:36:39 +00:00
|
|
|
) pulumirpc.LanguageRuntimeServer {
|
2018-02-10 02:15:04 +00:00
|
|
|
return &nodeLanguageHost{
|
2018-02-12 03:22:23 +00:00
|
|
|
engineAddress: engineAddress,
|
|
|
|
tracing: tracing,
|
2024-03-28 15:44:25 +00:00
|
|
|
forceTsc: forceTsc,
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-08-13 22:40:25 +00:00
|
|
|
func compatibleVersions(a, b semver.Version) (bool, string) {
|
|
|
|
switch {
|
|
|
|
case a.Major == 0 && b.Major == 0:
|
|
|
|
// If both major versions are pre-1.0, we require that the major and minor versions match.
|
|
|
|
if a.Minor != b.Minor {
|
|
|
|
return false, "Differing major or minor versions are not supported."
|
|
|
|
}
|
|
|
|
|
2020-03-30 22:10:42 +00:00
|
|
|
case a.Major >= 1 && a.Major <= 2 && b.Major >= 1 && b.Major <= 2:
|
|
|
|
// If both versions are 1.0<=v<=2.0, they are compatible.
|
|
|
|
|
|
|
|
case a.Major > 2 || b.Major > 2:
|
|
|
|
// If either version is post-2.0, we require that the major versions match.
|
2019-08-13 22:40:25 +00:00
|
|
|
if a.Major != b.Major {
|
|
|
|
return false, "Differing major versions are not supported."
|
|
|
|
}
|
|
|
|
|
|
|
|
case a.Major == 1 && b.Major == 0 && b.Minor == 17 || b.Major == 1 && a.Major == 0 && a.Minor == 17:
|
|
|
|
// If one version is pre-1.0 and the other is post-1.0, we unify 1.x.y and 0.17.z. This combination is legal.
|
|
|
|
|
|
|
|
default:
|
|
|
|
// All other combinations of versions are illegal.
|
|
|
|
return false, "Differing major or minor versions are not supported."
|
|
|
|
}
|
|
|
|
|
|
|
|
return true, ""
|
|
|
|
}
|
|
|
|
|
2018-02-12 03:22:23 +00:00
|
|
|
// GetRequiredPlugins computes the complete set of anticipated plugins required by a program.
|
|
|
|
func (host *nodeLanguageHost) GetRequiredPlugins(ctx context.Context,
|
2023-03-03 16:36:39 +00:00
|
|
|
req *pulumirpc.GetRequiredPluginsRequest,
|
|
|
|
) (*pulumirpc.GetRequiredPluginsResponse, error) {
|
2019-03-02 01:43:26 +00:00
|
|
|
// To get the plugins required by a program, find all node_modules/ packages that contain {
|
|
|
|
// "pulumi": true } inside of their package.json files. We begin this search in the same
|
|
|
|
// directory that contains the project. It's possible that a developer would do a
|
|
|
|
// `require("../../elsewhere")` and that we'd miss this as a dependency, however the solution
|
|
|
|
// for that is simple: install the package in the project root.
|
|
|
|
|
|
|
|
// Keep track of the versions of @pulumi/pulumi that are pulled in. If they differ on
|
|
|
|
// minor version, we will issue a warning to the user.
|
|
|
|
pulumiPackagePathToVersionMap := make(map[string]semver.Version)
|
2023-01-23 20:39:24 +00:00
|
|
|
plugins, err := getPluginsFromDir(
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
req.Info.ProgramDirectory,
|
2023-01-23 20:39:24 +00:00
|
|
|
pulumiPackagePathToVersionMap,
|
|
|
|
false, /*inNodeModules*/
|
|
|
|
make(map[string]struct{}))
|
2019-03-02 01:43:26 +00:00
|
|
|
|
|
|
|
if err == nil {
|
|
|
|
first := true
|
|
|
|
var firstPath string
|
|
|
|
var firstVersion semver.Version
|
|
|
|
for path, version := range pulumiPackagePathToVersionMap {
|
|
|
|
if first {
|
|
|
|
first = false
|
|
|
|
firstPath = path
|
|
|
|
firstVersion = version
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2019-08-13 22:40:25 +00:00
|
|
|
if ok, message := compatibleVersions(version, firstVersion); !ok {
|
2019-03-02 01:43:26 +00:00
|
|
|
fmt.Fprintf(os.Stderr,
|
2019-08-13 22:40:25 +00:00
|
|
|
`Found incompatible versions of @pulumi/pulumi. %s
|
2019-03-02 01:43:26 +00:00
|
|
|
Version %s referenced at %s
|
|
|
|
Version %s referenced at %s
|
2019-08-13 22:40:25 +00:00
|
|
|
`, message, firstVersion, firstPath, version, path)
|
2019-03-02 01:43:26 +00:00
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-12 03:22:23 +00:00
|
|
|
if err != nil {
|
2019-07-31 18:23:33 +00:00
|
|
|
logging.V(3).Infof("one or more errors while discovering plugins: %s", err)
|
2018-02-12 03:22:23 +00:00
|
|
|
}
|
|
|
|
return &pulumirpc.GetRequiredPluginsResponse{
|
|
|
|
Plugins: plugins,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// getPluginsFromDir enumerates all node_modules/ directories, deeply, and returns the fully concatenated results.
|
2019-03-02 01:43:26 +00:00
|
|
|
func getPluginsFromDir(
|
|
|
|
dir string, pulumiPackagePathToVersionMap map[string]semver.Version,
|
2023-03-03 16:36:39 +00:00
|
|
|
inNodeModules bool, visitedPaths map[string]struct{},
|
|
|
|
) ([]*pulumirpc.PluginDependency, error) {
|
2023-01-23 17:18:14 +00:00
|
|
|
// try to absolute the input path so visitedPaths can track it correctly
|
|
|
|
dir, err := filepath.Abs(dir)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("getting full path for plugin dir %s: %w", dir, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if _, has := visitedPaths[dir]; has {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
visitedPaths[dir] = struct{}{}
|
2019-03-02 01:43:26 +00:00
|
|
|
|
2022-09-10 07:22:11 +00:00
|
|
|
files, err := os.ReadDir(dir)
|
2018-02-12 03:22:23 +00:00
|
|
|
if err != nil {
|
2022-11-04 10:20:29 +00:00
|
|
|
return nil, fmt.Errorf("reading plugin dir %s: %w", dir, err)
|
2018-02-12 03:22:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var plugins []*pulumirpc.PluginDependency
|
Do not make errors during plugin discovery fatal
The plugin host can ask the language host to provide a list of
resource plugins that it thinks will be nessecary for use at
deployment time, so they can be eagerly loaded.
In NodeJS (the only language host that implements this RPC) This works
by walking the directory tree rooted at the CWD of the project,
looking for package.json files, parsing them and seeing it they have
some marker property set. If they do, we add information about them
which we return at the end of our walk.
If there is *any* error, the entire operation fails. We've seen a
bunch of cases where this happens:
- Broken symlinks written by some editors as part of autosave.
- Access denied errors when part of the tree is unwalkable (Eric ran
into this on Windows when he had a Pulumi program at the root of his
file system.
- Recusive symlinks leading to errors when trying to walk down the
infinite chain. (See #1634 for one such example).
The very frustrating thing about this is that when you hit an error
its not clear what is going on and fixing it can be non-trivial. Even
worse, in the normal case, all of these plugins are already installed
and could be loaded by the host (in the common case, plugins are
installed as a post install step when you run `npm install`) so if we
simply didn't do this check at all, things would work great.
This change does two things:
1. It does not stop at the first error we hit when discovering
plugins, instead we just record the error and continue.
2. Does not fail the overall operation if there was an error. Instead,
we return to the host what we have, which may be an incomplete view
of the world. We glog the errors we did discover for diagnostics if
we ever need them.
I believe that long term most of this code gets deleted anyway. I
expect we will move to a model long term where the engine faults in
the plugin (downloading it if needed) when a request for the plugin
arrives. But for now, we shouldn't block normal operations just
because we couldn't answer a question with full fidelity.
Fixes #1478
2018-08-08 22:48:36 +00:00
|
|
|
var allErrors *multierror.Error
|
2018-02-12 03:22:23 +00:00
|
|
|
for _, file := range files {
|
|
|
|
name := file.Name()
|
|
|
|
curr := filepath.Join(dir, name)
|
2023-01-23 17:18:14 +00:00
|
|
|
isDir := file.IsDir()
|
2018-02-19 18:58:03 +00:00
|
|
|
|
2023-01-23 17:18:14 +00:00
|
|
|
// if this is a symlink resolve it so our visitedPaths can track recursion
|
|
|
|
if (file.Type() & fs.ModeSymlink) != 0 {
|
2023-01-23 22:02:26 +00:00
|
|
|
symlink, err := filepath.EvalSymlinks(curr)
|
2023-01-23 17:18:14 +00:00
|
|
|
if err != nil {
|
|
|
|
allErrors = multierror.Append(allErrors, fmt.Errorf("resolving link in plugin dir %s: %w", curr, err))
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
curr = symlink
|
|
|
|
|
|
|
|
// And re-stat the directory to get the resolved mode bits
|
|
|
|
fi, err := os.Stat(curr)
|
|
|
|
if err != nil {
|
|
|
|
allErrors = multierror.Append(allErrors, err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
isDir = fi.IsDir()
|
2018-02-19 18:58:03 +00:00
|
|
|
}
|
2023-01-23 17:18:14 +00:00
|
|
|
|
|
|
|
if isDir {
|
2024-07-11 16:01:31 +00:00
|
|
|
// If a directory, recurse into it. However we have to take care to avoid recursing
|
|
|
|
// into nested policy packs. The plugins in a policy pack are not dependencies of the
|
|
|
|
// program, so we should not include them in the list of plugins to install.
|
|
|
|
policyPack, err := workspace.DetectPolicyPackPathFrom(curr)
|
|
|
|
if err != nil {
|
|
|
|
allErrors = multierror.Append(allErrors, err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
policyPackDir := filepath.Dir(policyPack)
|
|
|
|
if policyPack != "" && strings.Contains(curr, policyPackDir) {
|
|
|
|
// The path is within a policy pack, stop recursing.
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2019-03-02 01:43:26 +00:00
|
|
|
more, err := getPluginsFromDir(
|
2023-01-23 17:18:14 +00:00
|
|
|
curr,
|
|
|
|
pulumiPackagePathToVersionMap,
|
|
|
|
inNodeModules || filepath.Base(dir) == "node_modules",
|
|
|
|
visitedPaths)
|
2018-02-12 03:22:23 +00:00
|
|
|
if err != nil {
|
Do not make errors during plugin discovery fatal
The plugin host can ask the language host to provide a list of
resource plugins that it thinks will be nessecary for use at
deployment time, so they can be eagerly loaded.
In NodeJS (the only language host that implements this RPC) This works
by walking the directory tree rooted at the CWD of the project,
looking for package.json files, parsing them and seeing it they have
some marker property set. If they do, we add information about them
which we return at the end of our walk.
If there is *any* error, the entire operation fails. We've seen a
bunch of cases where this happens:
- Broken symlinks written by some editors as part of autosave.
- Access denied errors when part of the tree is unwalkable (Eric ran
into this on Windows when he had a Pulumi program at the root of his
file system.
- Recusive symlinks leading to errors when trying to walk down the
infinite chain. (See #1634 for one such example).
The very frustrating thing about this is that when you hit an error
its not clear what is going on and fixing it can be non-trivial. Even
worse, in the normal case, all of these plugins are already installed
and could be loaded by the host (in the common case, plugins are
installed as a post install step when you run `npm install`) so if we
simply didn't do this check at all, things would work great.
This change does two things:
1. It does not stop at the first error we hit when discovering
plugins, instead we just record the error and continue.
2. Does not fail the overall operation if there was an error. Instead,
we return to the host what we have, which may be an incomplete view
of the world. We glog the errors we did discover for diagnostics if
we ever need them.
I believe that long term most of this code gets deleted anyway. I
expect we will move to a model long term where the engine faults in
the plugin (downloading it if needed) when a request for the plugin
arrives. But for now, we shouldn't block normal operations just
because we couldn't answer a question with full fidelity.
Fixes #1478
2018-08-08 22:48:36 +00:00
|
|
|
allErrors = multierror.Append(allErrors, err)
|
2018-02-12 03:22:23 +00:00
|
|
|
}
|
2022-01-07 04:14:09 +00:00
|
|
|
// Even if there was an error, still append any plugins found in the dir.
|
2018-02-12 03:22:23 +00:00
|
|
|
plugins = append(plugins, more...)
|
2018-02-19 18:58:03 +00:00
|
|
|
} else if inNodeModules && name == "package.json" {
|
2018-02-12 03:22:23 +00:00
|
|
|
// if a package.json file within a node_modules package, parse it, and see if it's a source of plugins.
|
2023-01-06 22:39:16 +00:00
|
|
|
b, err := os.ReadFile(curr)
|
2018-02-12 03:22:23 +00:00
|
|
|
if err != nil {
|
2022-11-04 10:20:29 +00:00
|
|
|
allErrors = multierror.Append(allErrors, fmt.Errorf("reading package.json %s: %w", curr, err))
|
Do not make errors during plugin discovery fatal
The plugin host can ask the language host to provide a list of
resource plugins that it thinks will be nessecary for use at
deployment time, so they can be eagerly loaded.
In NodeJS (the only language host that implements this RPC) This works
by walking the directory tree rooted at the CWD of the project,
looking for package.json files, parsing them and seeing it they have
some marker property set. If they do, we add information about them
which we return at the end of our walk.
If there is *any* error, the entire operation fails. We've seen a
bunch of cases where this happens:
- Broken symlinks written by some editors as part of autosave.
- Access denied errors when part of the tree is unwalkable (Eric ran
into this on Windows when he had a Pulumi program at the root of his
file system.
- Recusive symlinks leading to errors when trying to walk down the
infinite chain. (See #1634 for one such example).
The very frustrating thing about this is that when you hit an error
its not clear what is going on and fixing it can be non-trivial. Even
worse, in the normal case, all of these plugins are already installed
and could be loaded by the host (in the common case, plugins are
installed as a post install step when you run `npm install`) so if we
simply didn't do this check at all, things would work great.
This change does two things:
1. It does not stop at the first error we hit when discovering
plugins, instead we just record the error and continue.
2. Does not fail the overall operation if there was an error. Instead,
we return to the host what we have, which may be an incomplete view
of the world. We glog the errors we did discover for diagnostics if
we ever need them.
I believe that long term most of this code gets deleted anyway. I
expect we will move to a model long term where the engine faults in
the plugin (downloading it if needed) when a request for the plugin
arrives. But for now, we shouldn't block normal operations just
because we couldn't answer a question with full fidelity.
Fixes #1478
2018-08-08 22:48:36 +00:00
|
|
|
continue
|
2018-02-12 03:22:23 +00:00
|
|
|
}
|
2019-03-02 01:43:26 +00:00
|
|
|
|
|
|
|
var info packageJSON
|
|
|
|
if err := json.Unmarshal(b, &info); err != nil {
|
2022-11-04 10:20:29 +00:00
|
|
|
allErrors = multierror.Append(allErrors, fmt.Errorf("unmarshaling package.json %s: %w", curr, err))
|
2019-03-02 01:43:26 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
if info.Name == "@pulumi/pulumi" {
|
|
|
|
version, err := semver.Parse(info.Version)
|
|
|
|
if err != nil {
|
|
|
|
allErrors = multierror.Append(
|
2022-11-04 10:20:29 +00:00
|
|
|
allErrors, fmt.Errorf("Could not understand version %s in '%s': %w", info.Version, curr, err))
|
2019-03-02 01:43:26 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
pulumiPackagePathToVersionMap[curr] = version
|
|
|
|
}
|
|
|
|
|
2019-05-30 22:02:46 +00:00
|
|
|
ok, name, version, server, err := getPackageInfo(info)
|
2018-02-12 03:22:23 +00:00
|
|
|
if err != nil {
|
2022-11-04 10:20:29 +00:00
|
|
|
allErrors = multierror.Append(allErrors, fmt.Errorf("unmarshaling package.json %s: %w", curr, err))
|
2018-02-12 03:22:23 +00:00
|
|
|
} else if ok {
|
|
|
|
plugins = append(plugins, &pulumirpc.PluginDependency{
|
|
|
|
Name: name,
|
|
|
|
Kind: "resource",
|
|
|
|
Version: version,
|
2019-05-30 22:02:46 +00:00
|
|
|
Server: server,
|
2018-02-12 03:22:23 +00:00
|
|
|
})
|
|
|
|
}
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
|
|
|
}
|
Do not make errors during plugin discovery fatal
The plugin host can ask the language host to provide a list of
resource plugins that it thinks will be nessecary for use at
deployment time, so they can be eagerly loaded.
In NodeJS (the only language host that implements this RPC) This works
by walking the directory tree rooted at the CWD of the project,
looking for package.json files, parsing them and seeing it they have
some marker property set. If they do, we add information about them
which we return at the end of our walk.
If there is *any* error, the entire operation fails. We've seen a
bunch of cases where this happens:
- Broken symlinks written by some editors as part of autosave.
- Access denied errors when part of the tree is unwalkable (Eric ran
into this on Windows when he had a Pulumi program at the root of his
file system.
- Recusive symlinks leading to errors when trying to walk down the
infinite chain. (See #1634 for one such example).
The very frustrating thing about this is that when you hit an error
its not clear what is going on and fixing it can be non-trivial. Even
worse, in the normal case, all of these plugins are already installed
and could be loaded by the host (in the common case, plugins are
installed as a post install step when you run `npm install`) so if we
simply didn't do this check at all, things would work great.
This change does two things:
1. It does not stop at the first error we hit when discovering
plugins, instead we just record the error and continue.
2. Does not fail the overall operation if there was an error. Instead,
we return to the host what we have, which may be an incomplete view
of the world. We glog the errors we did discover for diagnostics if
we ever need them.
I believe that long term most of this code gets deleted anyway. I
expect we will move to a model long term where the engine faults in
the plugin (downloading it if needed) when a request for the plugin
arrives. But for now, we shouldn't block normal operations just
because we couldn't answer a question with full fidelity.
Fixes #1478
2018-08-08 22:48:36 +00:00
|
|
|
return plugins, allErrors.ErrorOrNil()
|
2018-02-12 03:22:23 +00:00
|
|
|
}
|
2018-02-10 02:15:04 +00:00
|
|
|
|
2018-02-12 03:22:23 +00:00
|
|
|
// packageJSON is the minimal amount of package.json information we care about.
|
|
|
|
type packageJSON struct {
|
2022-08-15 13:55:04 +00:00
|
|
|
Name string `json:"name"`
|
|
|
|
Version string `json:"version"`
|
|
|
|
Pulumi plugin.PulumiPluginJSON `json:"pulumi"`
|
|
|
|
Main string `json:"main"`
|
|
|
|
Dependencies map[string]string `json:"dependencies"`
|
|
|
|
DevDependencies map[string]string `json:"devDependencies"`
|
2018-02-12 03:22:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// getPackageInfo returns a bool indicating whether the given package.json package has an associated Pulumi
|
2022-09-13 14:40:09 +00:00
|
|
|
// resource provider plugin. If it does, three strings are returned, the plugin name, and its semantic version and
|
2019-05-30 22:02:46 +00:00
|
|
|
// an optional server that can be used to download the plugin (this may be empty, in which case the "default" location
|
|
|
|
// should be used).
|
|
|
|
func getPackageInfo(info packageJSON) (bool, string, string, string, error) {
|
2018-02-12 03:22:23 +00:00
|
|
|
if info.Pulumi.Resource {
|
|
|
|
name, err := getPluginName(info)
|
|
|
|
if err != nil {
|
2019-05-30 22:02:46 +00:00
|
|
|
return false, "", "", "", err
|
2018-02-12 03:22:23 +00:00
|
|
|
}
|
|
|
|
version, err := getPluginVersion(info)
|
|
|
|
if err != nil {
|
2019-05-30 22:02:46 +00:00
|
|
|
return false, "", "", "", err
|
2018-02-12 03:22:23 +00:00
|
|
|
}
|
2019-05-30 22:02:46 +00:00
|
|
|
return true, name, version, info.Pulumi.Server, nil
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
|
|
|
|
2019-05-30 22:02:46 +00:00
|
|
|
return false, "", "", "", nil
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
|
|
|
|
2018-02-12 03:22:23 +00:00
|
|
|
// getPluginName takes a parsed package.json file and returns the corresponding Pulumi plugin name.
|
|
|
|
func getPluginName(info packageJSON) (string, error) {
|
2020-12-04 03:22:16 +00:00
|
|
|
// If it's specified in the "pulumi" section, return it as-is.
|
|
|
|
if info.Pulumi.Name != "" {
|
|
|
|
return info.Pulumi.Name, nil
|
|
|
|
}
|
|
|
|
|
2022-09-13 14:40:09 +00:00
|
|
|
// Otherwise, derive it from the top-level package name,
|
|
|
|
// only if it has @pulumi scope, otherwise fail.
|
2018-02-12 03:22:23 +00:00
|
|
|
name := info.Name
|
|
|
|
if name == "" {
|
|
|
|
return "", errors.New("missing expected \"name\" property")
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
|
|
|
|
2022-09-13 14:40:09 +00:00
|
|
|
// If the name has a @pulumi scope, we will just use its simple name. Otherwise, we use the fully scoped name.
|
2018-02-12 03:22:23 +00:00
|
|
|
// We do trim the leading @, however, since Pulumi resource providers do not use the same NPM convention.
|
|
|
|
if strings.Index(name, "@pulumi/") == 0 {
|
|
|
|
return name[strings.IndexRune(name, '/')+1:], nil
|
|
|
|
}
|
2022-09-13 14:40:09 +00:00
|
|
|
|
|
|
|
// if the package name does not start with @pulumi it means that it is a third-party package
|
|
|
|
// third-party packages _MUST_ have the plugin name in package.json in the pulumi section
|
|
|
|
// {
|
|
|
|
// "name": "@third-party-package"
|
|
|
|
// "pulumi": {
|
|
|
|
// "name": "<plugin name>"
|
|
|
|
// }
|
|
|
|
// }
|
|
|
|
|
2022-11-04 10:20:29 +00:00
|
|
|
return "", fmt.Errorf("Missing property \"name\" for the third-party plugin '%v' "+
|
2022-09-13 14:40:09 +00:00
|
|
|
"inside package.json under the \"pulumi\" section.", name)
|
2018-02-12 03:22:23 +00:00
|
|
|
}
|
2018-02-10 02:15:04 +00:00
|
|
|
|
2018-02-12 03:22:23 +00:00
|
|
|
// getPluginVersion takes a parsed package.json file and returns the semantic version of the Pulumi plugin.
|
|
|
|
func getPluginVersion(info packageJSON) (string, error) {
|
2020-12-04 03:22:16 +00:00
|
|
|
// See if it's specified in the "pulumi" section.
|
|
|
|
version := info.Pulumi.Version
|
2018-02-12 03:22:23 +00:00
|
|
|
if version == "" {
|
2020-12-04 03:22:16 +00:00
|
|
|
// If not, use the top-level package version.
|
|
|
|
version = info.Version
|
|
|
|
if version == "" {
|
|
|
|
return "", errors.New("Missing expected \"version\" property")
|
|
|
|
}
|
2018-02-12 03:22:23 +00:00
|
|
|
}
|
|
|
|
if strings.IndexRune(version, 'v') != 0 {
|
2024-04-19 06:20:33 +00:00
|
|
|
return "v" + version, nil
|
2018-02-12 03:22:23 +00:00
|
|
|
}
|
|
|
|
return version, nil
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
|
|
|
|
2019-10-15 05:08:06 +00:00
|
|
|
// When talking to the nodejs runtime we have three parties involved:
|
|
|
|
//
|
|
|
|
// Engine Monitor <==> Language Host (this code) <==> NodeJS sdk runtime.
|
|
|
|
//
|
|
|
|
// Instead of having the NodeJS sdk runtime communicating directly with the Engine Monitor, we
|
|
|
|
// instead have it communicate with us and we send all those messages to the real engine monitor
|
|
|
|
// itself. We do that by having ourselves launch our own grpc monitor server and passing the
|
|
|
|
// address of it to the NodeJS runtime. As far as the NodeJS sdk runtime is concerned, it is
|
|
|
|
// communicating directly with the engine.
|
|
|
|
//
|
|
|
|
// When NodeJS then communicates back with us over our server, we then forward the messages
|
|
|
|
// along untouched to the Engine Monitor. However, we also open an additional *non-grpc*
|
|
|
|
// channel to allow the sdk runtime to send us messages on. Specifically, this non-grpc channel
|
|
|
|
// is used entirely to allow the sdk runtime to make 'invoke' calls in a synchronous fashion.
|
|
|
|
// This is accomplished by avoiding grpc entirely (which has no facility for synchronous rpc
|
|
|
|
// calls), and instead operating over a pair of files coordinated between us and the sdk
|
|
|
|
// runtime. One file is used by it to send us messages, and one file is used by us to send
|
|
|
|
// messages back. Because these are just files, nodejs natively supports allowing both sides to
|
|
|
|
// read and write from each synchronously.
|
|
|
|
//
|
|
|
|
// When we receive the sync-invoke messages from the nodejs sdk we deserialize things off of the
|
|
|
|
// file and then make a synchronous call to the real engine `invoke` monitor endpoint. Unlike
|
|
|
|
// nodejs, we have no problem calling this synchronously, and can block until we get the
|
|
|
|
// response which we can then synchronously send to nodejs.
|
|
|
|
|
2022-10-09 14:58:33 +00:00
|
|
|
// Run is the RPC endpoint for LanguageRuntimeServer::Run
|
2018-02-10 02:15:04 +00:00
|
|
|
func (host *nodeLanguageHost) Run(ctx context.Context, req *pulumirpc.RunRequest) (*pulumirpc.RunResponse, error) {
|
2019-10-15 05:08:06 +00:00
|
|
|
tracingSpan := opentracing.SpanFromContext(ctx)
|
|
|
|
|
|
|
|
// Make a connection to the real monitor that we will forward messages to.
|
2020-04-20 22:25:51 +00:00
|
|
|
conn, err := grpc.Dial(
|
|
|
|
req.GetMonitorAddress(),
|
2023-01-11 19:54:31 +00:00
|
|
|
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
2020-04-20 22:25:51 +00:00
|
|
|
rpcutil.GrpcChannelOptions(),
|
|
|
|
)
|
2018-02-10 02:15:04 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2019-10-15 05:08:06 +00:00
|
|
|
// Make a client around that connection. We can then make our own server that will act as a
|
|
|
|
// monitor for the sdk and forward to the real monitor.
|
|
|
|
target := pulumirpc.NewResourceMonitorClient(conn)
|
2018-04-30 23:10:01 +00:00
|
|
|
|
2019-10-15 05:08:06 +00:00
|
|
|
// Channel to control the server lifetime. Once `Run` finishes, we'll shutdown the server.
|
|
|
|
serverCancel := make(chan bool)
|
|
|
|
defer func() {
|
|
|
|
serverCancel <- true
|
|
|
|
close(serverCancel)
|
|
|
|
}()
|
|
|
|
|
|
|
|
// Launch the rpc server giving it the real monitor to forward messages to.
|
2022-11-01 15:15:09 +00:00
|
|
|
handle, err := rpcutil.ServeWithOptions(rpcutil.ServeOptions{
|
|
|
|
Cancel: serverCancel,
|
|
|
|
Init: func(srv *grpc.Server) error {
|
2022-12-14 19:20:26 +00:00
|
|
|
pulumirpc.RegisterResourceMonitorServer(srv, &monitorProxy{target: target})
|
2019-10-15 05:08:06 +00:00
|
|
|
return nil
|
|
|
|
},
|
2022-11-01 15:15:09 +00:00
|
|
|
Options: rpcutil.OpenTracingServerInterceptorOptions(tracingSpan),
|
|
|
|
})
|
2019-10-15 05:08:06 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2018-06-25 05:47:54 +00:00
|
|
|
}
|
|
|
|
|
2019-10-15 05:08:06 +00:00
|
|
|
// Create the pipes we'll use to communicate synchronously with the nodejs process. Once we're
|
|
|
|
// done using the pipes clean them up so we don't leave anything around in the user file system.
|
|
|
|
pipes, pipesDone, err := createAndServePipes(ctx, target)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
2023-02-27 20:28:05 +00:00
|
|
|
defer pipes.shutdown()
|
2018-02-10 02:15:04 +00:00
|
|
|
|
2022-04-03 14:54:59 +00:00
|
|
|
nodeBin, err := exec.LookPath("node")
|
|
|
|
if err != nil {
|
2024-05-10 15:16:13 +00:00
|
|
|
return &pulumirpc.RunResponse{Error: "could not find node on the $PATH: " + err.Error()}, nil
|
2022-04-03 14:54:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
runPath := os.Getenv("PULUMI_LANGUAGE_NODEJS_RUN_PATH")
|
|
|
|
if runPath == "" {
|
|
|
|
runPath = defaultRunPath
|
|
|
|
}
|
|
|
|
|
2024-03-28 15:44:25 +00:00
|
|
|
// If we're forcing tsc the program directory for running is actually ./bin
|
|
|
|
programDirectory := req.Info.ProgramDirectory
|
|
|
|
if host.forceTsc {
|
|
|
|
req.Info.ProgramDirectory = filepath.Join(programDirectory, "bin")
|
|
|
|
}
|
|
|
|
|
|
|
|
runPath, err = locateModule(ctx, runPath, programDirectory, nodeBin)
|
2022-04-03 14:54:59 +00:00
|
|
|
if err != nil {
|
2024-05-10 15:16:13 +00:00
|
|
|
return &pulumirpc.RunResponse{Error: err.Error()}, nil
|
2022-04-03 14:54:59 +00:00
|
|
|
}
|
|
|
|
|
2023-09-03 07:26:15 +00:00
|
|
|
// Channel producing the final response we want to issue to our caller. Will get the result of
|
|
|
|
// the actual nodejs process we launch, or any results caused by errors in our server/pipes.
|
|
|
|
responseChannel := make(chan *pulumirpc.RunResponse)
|
2019-10-15 05:08:06 +00:00
|
|
|
// now, launch the nodejs process and actually run the user code in it.
|
2023-09-03 07:26:15 +00:00
|
|
|
go func() {
|
|
|
|
defer close(responseChannel)
|
|
|
|
responseChannel <- host.execNodejs(
|
|
|
|
ctx, req, nodeBin, runPath,
|
|
|
|
fmt.Sprintf("127.0.0.1:%d", handle.Port), pipes.directory())
|
|
|
|
}()
|
2019-10-15 05:08:06 +00:00
|
|
|
|
|
|
|
// Wait for one of our launched goroutines to signal that we're done. This might be our proxy
|
|
|
|
// (in the case of errors), or the launched nodejs completing (either successfully, or with
|
|
|
|
// errors).
|
2023-09-03 07:26:15 +00:00
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case err := <-handle.Done:
|
|
|
|
if err != nil {
|
|
|
|
return &pulumirpc.RunResponse{Error: err.Error()}, nil
|
|
|
|
}
|
|
|
|
case err := <-pipesDone:
|
|
|
|
if err != nil {
|
|
|
|
return &pulumirpc.RunResponse{Error: err.Error()}, nil
|
|
|
|
}
|
|
|
|
case response := <-responseChannel:
|
|
|
|
return response, nil
|
|
|
|
}
|
|
|
|
}
|
2019-10-15 05:08:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Launch the nodejs process and wait for it to complete. Report success or any errors using the
|
|
|
|
// `responseChannel` arg.
|
2023-09-03 07:26:15 +00:00
|
|
|
func (host *nodeLanguageHost) execNodejs(ctx context.Context, req *pulumirpc.RunRequest,
|
2023-03-03 16:36:39 +00:00
|
|
|
nodeBin, runPath, address, pipesDirectory string,
|
2023-09-03 07:26:15 +00:00
|
|
|
) *pulumirpc.RunResponse {
|
2019-10-15 05:08:06 +00:00
|
|
|
// Actually launch nodejs and process the result of it into an appropriate response object.
|
2023-09-03 07:26:15 +00:00
|
|
|
args := host.constructArguments(req, runPath, address, pipesDirectory)
|
|
|
|
config, err := host.constructConfig(req)
|
|
|
|
if err != nil {
|
|
|
|
err = fmt.Errorf("failed to serialize configuration: %w", err)
|
|
|
|
return &pulumirpc.RunResponse{Error: err.Error()}
|
|
|
|
}
|
|
|
|
configSecretKeys, err := host.constructConfigSecretKeys(req)
|
|
|
|
if err != nil {
|
|
|
|
err = fmt.Errorf("failed to serialize configuration secret keys: %w", err)
|
|
|
|
return &pulumirpc.RunResponse{Error: err.Error()}
|
|
|
|
}
|
2019-10-15 05:08:06 +00:00
|
|
|
|
2023-09-03 07:26:15 +00:00
|
|
|
env := os.Environ()
|
|
|
|
env = append(env, pulumiConfigVar+"="+config)
|
|
|
|
env = append(env, pulumiConfigSecretKeysVar+"="+configSecretKeys)
|
2019-03-20 18:54:32 +00:00
|
|
|
|
2024-06-21 11:35:06 +00:00
|
|
|
opts, err := parseOptions(req.Info.Options.AsMap())
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
if err != nil {
|
|
|
|
return &pulumirpc.RunResponse{Error: err.Error()}
|
|
|
|
}
|
|
|
|
|
|
|
|
if opts.typescript {
|
2023-09-03 07:26:15 +00:00
|
|
|
env = append(env, "PULUMI_NODEJS_TYPESCRIPT=true")
|
|
|
|
}
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
if opts.tsconfigpath != "" {
|
|
|
|
env = append(env, "PULUMI_NODEJS_TSCONFIG_PATH="+opts.tsconfigpath)
|
2023-09-03 07:26:15 +00:00
|
|
|
}
|
2019-10-15 05:08:06 +00:00
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
nodeargs, err := shlex.Split(opts.nodeargs)
|
2023-09-03 07:26:15 +00:00
|
|
|
if err != nil {
|
|
|
|
return &pulumirpc.RunResponse{Error: err.Error()}
|
|
|
|
}
|
|
|
|
nodeargs = append(nodeargs, args...)
|
2022-01-05 02:54:38 +00:00
|
|
|
|
2023-09-03 07:26:15 +00:00
|
|
|
if logging.V(5) {
|
|
|
|
commandStr := strings.Join(nodeargs, " ")
|
|
|
|
logging.V(5).Infoln("Language host launching process: ", nodeBin, commandStr)
|
|
|
|
}
|
2019-10-15 05:08:06 +00:00
|
|
|
|
2023-09-03 07:26:15 +00:00
|
|
|
// Now simply spawn a process to execute the requested program, wiring up stdout/stderr directly.
|
|
|
|
var errResult string
|
|
|
|
// #nosec G204
|
|
|
|
cmd := exec.Command(nodeBin, nodeargs...)
|
2024-07-23 16:10:43 +00:00
|
|
|
// Copy cmd.Stdout to os.Stdout. Nodejs sometimes changes the blocking mode of its stdout/stderr,
|
|
|
|
// so it's unsafe to assign cmd.Stdout directly to os.Stdout. See the description of
|
|
|
|
// `runWithOutput` for more details.
|
|
|
|
cmd.Stdout = struct{ io.Writer }{os.Stdout}
|
|
|
|
r, w := io.Pipe()
|
|
|
|
cmd.Stderr = w
|
|
|
|
// Get a duplicate reader of stderr so that we can both scan it and write it to os.Stderr.
|
|
|
|
stderrDup := io.TeeReader(r, os.Stderr)
|
|
|
|
sniffer := newOOMSniffer()
|
|
|
|
sniffer.Scan(stderrDup)
|
|
|
|
|
2023-09-03 07:26:15 +00:00
|
|
|
cmd.Env = env
|
2019-10-15 05:08:06 +00:00
|
|
|
|
2023-09-03 07:26:15 +00:00
|
|
|
tracingSpan, _ := opentracing.StartSpanFromContext(ctx,
|
|
|
|
"execNodejs",
|
|
|
|
opentracing.Tag{Key: "component", Value: "exec.Command"},
|
|
|
|
opentracing.Tag{Key: "command", Value: nodeBin},
|
|
|
|
opentracing.Tag{Key: "args", Value: nodeargs})
|
|
|
|
defer tracingSpan.Finish()
|
|
|
|
|
|
|
|
if err := cmd.Run(); err != nil {
|
|
|
|
// NodeJS stdout is complicated enough that we should explicitly flush stdout and stderr here. NodeJS does
|
|
|
|
// process writes using console.out and console.err synchronously, but it does not process writes using
|
|
|
|
// `process.stdout.write` or `process.stderr.write` synchronously, and it is possible that there exist unflushed
|
|
|
|
// writes on those file descriptors at the time that the Node process exits.
|
|
|
|
//
|
|
|
|
// Because of this, we explicitly flush stdout and stderr so that we are absolutely sure that we capture any
|
|
|
|
// error messages in the engine.
|
|
|
|
contract.IgnoreError(os.Stdout.Sync())
|
|
|
|
contract.IgnoreError(os.Stderr.Sync())
|
|
|
|
if exiterr, ok := err.(*exec.ExitError); ok {
|
|
|
|
// If the program ran, but exited with a non-zero error code. This will happen often,
|
|
|
|
// since user errors will trigger this. So, the error message should look as nice as
|
|
|
|
// possible.
|
|
|
|
switch code := exiterr.ExitCode(); code {
|
|
|
|
case 0:
|
|
|
|
// This really shouldn't happen, but if it does, we don't want to render "non-zero exit code"
|
|
|
|
err = fmt.Errorf("Program exited unexpectedly: %w", exiterr)
|
|
|
|
case nodeJSProcessExitedAfterShowingUserActionableMessage:
|
|
|
|
// Check if we got special exit code that means "we already gave the user an
|
|
|
|
// actionable message". In that case, we can simply bail out and terminate `pulumi`
|
|
|
|
// without showing any more messages.
|
|
|
|
return &pulumirpc.RunResponse{Error: "", Bail: true}
|
|
|
|
default:
|
|
|
|
err = fmt.Errorf("Program exited with non-zero exit code: %d", code)
|
2024-07-23 16:10:43 +00:00
|
|
|
sniffer.Wait()
|
|
|
|
if sniffer.Detected() {
|
|
|
|
err = fmt.Errorf("Program exited with non-zero exit code: %d. %s", code, sniffer.Message())
|
|
|
|
}
|
2023-09-03 07:26:15 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// Otherwise, we didn't even get to run the program. This ought to never happen unless there's
|
|
|
|
// a bug or system condition that prevented us from running the language exec. Issue a scarier error.
|
|
|
|
err = fmt.Errorf("Problem executing program (could not run language executor): %w", err)
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
|
|
|
|
2023-09-03 07:26:15 +00:00
|
|
|
errResult = err.Error()
|
|
|
|
}
|
2018-02-10 02:15:04 +00:00
|
|
|
|
2019-10-15 05:08:06 +00:00
|
|
|
// notify our caller of the response we got from the nodejs process. Note: this is done
|
|
|
|
// unilaterally. this is how we signal to nodeLanguageHost.Run that we are done and it can
|
|
|
|
// return to its caller.
|
2023-09-03 07:26:15 +00:00
|
|
|
return &pulumirpc.RunResponse{Error: errResult}
|
2018-02-10 02:15:04 +00:00
|
|
|
}
|
|
|
|
|
2018-02-12 03:22:23 +00:00
|
|
|
// constructArguments constructs a command-line for `pulumi-language-nodejs`
|
|
|
|
// by enumerating all of the optional and non-optional arguments present
|
|
|
|
// in a RunRequest.
|
2022-04-03 14:54:59 +00:00
|
|
|
func (host *nodeLanguageHost) constructArguments(
|
2023-03-03 16:36:39 +00:00
|
|
|
req *pulumirpc.RunRequest, runPath, address, pipesDirectory string,
|
|
|
|
) []string {
|
2022-04-03 14:54:59 +00:00
|
|
|
args := []string{runPath}
|
2018-02-12 03:22:23 +00:00
|
|
|
maybeAppendArg := func(k, v string) {
|
|
|
|
if v != "" {
|
|
|
|
args = append(args, "--"+k, v)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-10-15 05:08:06 +00:00
|
|
|
maybeAppendArg("monitor", address)
|
2018-02-12 03:22:23 +00:00
|
|
|
maybeAppendArg("engine", host.engineAddress)
|
2019-10-15 05:08:06 +00:00
|
|
|
maybeAppendArg("sync", pipesDirectory)
|
2022-08-31 17:24:25 +00:00
|
|
|
maybeAppendArg("organization", req.GetOrganization())
|
2018-02-12 03:22:23 +00:00
|
|
|
maybeAppendArg("project", req.GetProject())
|
|
|
|
maybeAppendArg("stack", req.GetStack())
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
maybeAppendArg("pwd", req.Info.ProgramDirectory)
|
2018-02-12 03:22:23 +00:00
|
|
|
if req.GetDryRun() {
|
|
|
|
args = append(args, "--dry-run")
|
|
|
|
}
|
|
|
|
|
2023-12-12 12:19:42 +00:00
|
|
|
maybeAppendArg("query-mode", strconv.FormatBool(req.GetQueryMode()))
|
|
|
|
maybeAppendArg("parallel", strconv.Itoa(int(req.GetParallel())))
|
2018-02-12 03:22:23 +00:00
|
|
|
maybeAppendArg("tracing", host.tracing)
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
|
|
|
// The engine should always pass a name for entry point, even if its just "." for the program directory.
|
|
|
|
args = append(args, req.Info.EntryPoint)
|
2018-02-12 03:22:23 +00:00
|
|
|
|
|
|
|
args = append(args, req.GetArgs()...)
|
|
|
|
return args
|
|
|
|
}
|
|
|
|
|
2019-12-16 22:51:02 +00:00
|
|
|
// constructConfig JSON-serializes the configuration data given as part of
|
2018-02-12 03:22:23 +00:00
|
|
|
// a RunRequest.
|
|
|
|
func (host *nodeLanguageHost) constructConfig(req *pulumirpc.RunRequest) (string, error) {
|
|
|
|
configMap := req.GetConfig()
|
|
|
|
if configMap == nil {
|
|
|
|
return "{}", nil
|
|
|
|
}
|
|
|
|
|
2018-03-02 23:23:59 +00:00
|
|
|
// While we transition from the old format for config keys (<package>:config:<name> to <package>:<name>), we want
|
|
|
|
// to support the newest version of the langhost running older packages, so the config bag we present to them looks
|
|
|
|
// like the old world. Newer versions of the @pulumi/pulumi package handle both formats and when we stop supporting
|
|
|
|
// older versions, we can remove this code.
|
|
|
|
transformedConfig := make(map[string]string, len(configMap))
|
|
|
|
for k, v := range configMap {
|
|
|
|
pk, err := config.ParseKey(k)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
transformedConfig[pk.Namespace()+":config:"+pk.Name()] = v
|
|
|
|
}
|
|
|
|
|
2018-03-14 04:52:16 +00:00
|
|
|
configJSON, err := json.Marshal(transformedConfig)
|
2018-02-12 03:22:23 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
return string(configJSON), nil
|
|
|
|
}
|
|
|
|
|
2021-05-18 16:48:08 +00:00
|
|
|
// constructConfigSecretKeys JSON-serializes the list of keys that contain secret values given as part of
|
|
|
|
// a RunRequest.
|
|
|
|
func (host *nodeLanguageHost) constructConfigSecretKeys(req *pulumirpc.RunRequest) (string, error) {
|
|
|
|
configSecretKeys := req.GetConfigSecretKeys()
|
|
|
|
if configSecretKeys == nil {
|
|
|
|
return "[]", nil
|
|
|
|
}
|
|
|
|
|
|
|
|
configSecretKeysJSON, err := json.Marshal(configSecretKeys)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
return string(configSecretKeysJSON), nil
|
|
|
|
}
|
|
|
|
|
2024-01-17 09:35:20 +00:00
|
|
|
func (host *nodeLanguageHost) GetPluginInfo(ctx context.Context, req *emptypb.Empty) (*pulumirpc.PluginInfo, error) {
|
2018-02-10 02:15:04 +00:00
|
|
|
return &pulumirpc.PluginInfo{
|
|
|
|
Version: version.Version,
|
|
|
|
}, nil
|
|
|
|
}
|
2022-04-03 14:54:59 +00:00
|
|
|
|
|
|
|
func (host *nodeLanguageHost) InstallDependencies(
|
2023-03-03 16:36:39 +00:00
|
|
|
req *pulumirpc.InstallDependenciesRequest, server pulumirpc.LanguageRuntime_InstallDependenciesServer,
|
|
|
|
) error {
|
2022-10-04 08:58:01 +00:00
|
|
|
closer, stdout, stderr, err := rpcutil.MakeInstallDependenciesStreams(server, req.IsTerminal)
|
2022-04-03 14:54:59 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
// best effort close, but we try an explicit close and error check at the end as well
|
|
|
|
defer closer.Close()
|
|
|
|
|
2022-09-12 21:42:27 +00:00
|
|
|
tracingSpan, ctx := opentracing.StartSpanFromContext(server.Context(), "npm-install")
|
|
|
|
defer tracingSpan.Finish()
|
|
|
|
|
2022-04-03 14:54:59 +00:00
|
|
|
stdout.Write([]byte("Installing dependencies...\n\n"))
|
|
|
|
|
2024-02-16 08:25:12 +00:00
|
|
|
workspaceRoot := req.Info.ProgramDirectory
|
|
|
|
newWorkspaceRoot, err := npm.FindWorkspaceRoot(req.Info.ProgramDirectory)
|
2022-04-03 14:54:59 +00:00
|
|
|
if err != nil {
|
2024-02-16 08:25:12 +00:00
|
|
|
// If we are not in a npm/yarn workspace, we will keep the current directory as the root.
|
|
|
|
if !errors.Is(err, npm.ErrNotInWorkspace) {
|
|
|
|
return fmt.Errorf("failure while trying to find workspace root: %w", err)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
stdout.Write([]byte(fmt.Sprintf("Detected workspace root at %s\n", newWorkspaceRoot)))
|
|
|
|
workspaceRoot = newWorkspaceRoot
|
|
|
|
}
|
|
|
|
|
2024-06-21 11:35:06 +00:00
|
|
|
opts, err := parseOptions(req.Info.Options.AsMap())
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to parse options: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err = npm.Install(ctx, opts.packagemanager, workspaceRoot, false /*production*/, stdout, stderr)
|
2024-02-16 08:25:12 +00:00
|
|
|
if err != nil {
|
2024-06-28 23:22:17 +00:00
|
|
|
return err
|
2022-04-03 14:54:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
stdout.Write([]byte("Finished installing dependencies\n\n"))
|
|
|
|
|
2024-03-28 15:44:25 +00:00
|
|
|
if host.forceTsc {
|
|
|
|
// If we're forcing tsc for conformance testing this is our chance to run it before actually running the program.
|
|
|
|
// We probably want to see about making something like this an explicit "pulumi build" step, but for now shim'ing this
|
|
|
|
// in here works well enough for conformance testing.
|
|
|
|
tscCmd := exec.Command("npx", "tsc")
|
|
|
|
tscCmd.Dir = req.Info.ProgramDirectory
|
2024-07-23 16:10:43 +00:00
|
|
|
if err := runWithOutput(tscCmd, os.Stdout, os.Stderr); err != nil {
|
2024-03-28 15:44:25 +00:00
|
|
|
return fmt.Errorf("failed to run tsc: %w", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
all: Fix revive issues
Fixes the following issues found by revive
included in the latest release of golangci-lint.
Full list of issues:
**pkg**
```
backend/display/object_diff.go:47:10: superfluous-else: if block ends with a break statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
backend/display/object_diff.go:716:12: redefines-builtin-id: redefinition of the built-in function delete (revive)
backend/display/object_diff.go:742:14: redefines-builtin-id: redefinition of the built-in function delete (revive)
backend/display/object_diff.go:983:10: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (revive)
backend/httpstate/backend.go:1814:4: redefines-builtin-id: redefinition of the built-in function cap (revive)
backend/httpstate/backend.go:1824:5: redefines-builtin-id: redefinition of the built-in function cap (revive)
backend/httpstate/client/client.go:444:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
backend/httpstate/client/client.go:455:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
cmd/pulumi/org.go:113:4: if-return: redundant if ...; err != nil check, just return error instead. (revive)
cmd/pulumi/util.go:216:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
codegen/docs/gen.go:428:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
codegen/hcl2/model/expression.go:2151:5: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:151:2: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:329:3: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:381:5: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/nodejs/gen.go:1367:5: redefines-builtin-id: redefinition of the built-in function copy (revive)
codegen/python/gen_program_expressions.go:136:2: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/python/gen_program_expressions.go:142:3: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/report/report.go:126:6: redefines-builtin-id: redefinition of the built-in function panic (revive)
codegen/schema/docs_test.go:210:10: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
codegen/schema/schema.go:790:2: redefines-builtin-id: redefinition of the built-in type any (revive)
codegen/schema/schema.go:793:4: redefines-builtin-id: redefinition of the built-in type any (revive)
resource/deploy/plan.go:506:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
resource/deploy/snapshot_test.go:59:3: redefines-builtin-id: redefinition of the built-in function copy (revive)
resource/deploy/state_builder.go:108:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
```
**sdk**
```
go/common/resource/plugin/context.go:142:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
go/common/resource/plugin/plugin.go:142:12: superfluous-else: if block ends with a break statement, so drop this else and outdent its block (revive)
go/common/resource/properties_diff.go:114:2: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:117:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:122:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:127:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:132:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/util/deepcopy/copy.go:30:1: redefines-builtin-id: redefinition of the built-in function copy (revive)
go/common/workspace/creds.go:242:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi-language-go/main.go:569:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi-language-go/main.go:706:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi/run_test.go:925:2: redefines-builtin-id: redefinition of the built-in type any (revive)
go/pulumi/run_test.go:933:3: redefines-builtin-id: redefinition of the built-in type any (revive)
nodejs/cmd/pulumi-language-nodejs/main.go:778:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/cmd/pulumi-language-python/main.go:1011:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/cmd/pulumi-language-python/main.go:863:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/python.go:230:2: redefines-builtin-id: redefinition of the built-in function print (revive)
```
**tests**
```
integration/integration_util_test.go:282:11: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
```
2023-03-20 23:48:02 +00:00
|
|
|
return closer.Close()
|
2022-04-03 14:54:59 +00:00
|
|
|
}
|
2022-07-25 11:35:16 +00:00
|
|
|
|
2024-06-17 17:10:55 +00:00
|
|
|
func (host *nodeLanguageHost) RuntimeOptionsPrompts(ctx context.Context,
|
|
|
|
req *pulumirpc.RuntimeOptionsRequest,
|
|
|
|
) (*pulumirpc.RuntimeOptionsResponse, error) {
|
2024-06-21 11:35:06 +00:00
|
|
|
var prompts []*pulumirpc.RuntimeOptionPrompt
|
|
|
|
rawOpts := req.Info.Options.AsMap()
|
|
|
|
|
|
|
|
if _, hasPackagemanager := rawOpts["packagemanager"]; !hasPackagemanager {
|
|
|
|
prompts = append(prompts, &pulumirpc.RuntimeOptionPrompt{
|
|
|
|
Key: "packagemanager",
|
2024-06-28 23:17:42 +00:00
|
|
|
Description: "The package manager to use for installing dependencies",
|
2024-06-21 11:35:06 +00:00
|
|
|
PromptType: pulumirpc.RuntimeOptionPrompt_STRING,
|
2024-06-28 23:21:55 +00:00
|
|
|
Choices: plugin.MakeExecutablePromptChoices("npm", "pnpm", "yarn"),
|
2024-06-21 11:35:06 +00:00
|
|
|
Default: &pulumirpc.RuntimeOptionPrompt_RuntimeOptionValue{
|
|
|
|
PromptType: pulumirpc.RuntimeOptionPrompt_STRING,
|
|
|
|
StringValue: "npm",
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
return &pulumirpc.RuntimeOptionsResponse{
|
|
|
|
Prompts: prompts,
|
|
|
|
}, nil
|
2024-06-17 17:10:55 +00:00
|
|
|
}
|
|
|
|
|
2024-06-06 08:21:46 +00:00
|
|
|
func (host *nodeLanguageHost) About(ctx context.Context,
|
|
|
|
req *pulumirpc.AboutRequest,
|
|
|
|
) (*pulumirpc.AboutResponse, error) {
|
2022-08-15 13:55:04 +00:00
|
|
|
getResponse := func(execString string, args ...string) (string, string, error) {
|
|
|
|
ex, err := executable.FindExecutable(execString)
|
|
|
|
if err != nil {
|
|
|
|
return "", "", fmt.Errorf("could not find executable '%s': %w", execString, err)
|
|
|
|
}
|
|
|
|
cmd := exec.Command(ex, args...)
|
|
|
|
var out []byte
|
|
|
|
if out, err = cmd.Output(); err != nil {
|
|
|
|
cmd := ex
|
|
|
|
if len(args) != 0 {
|
|
|
|
cmd += " " + strings.Join(args, " ")
|
|
|
|
}
|
|
|
|
return "", "", fmt.Errorf("failed to execute '%s'", cmd)
|
|
|
|
}
|
|
|
|
return ex, strings.TrimSpace(string(out)), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
node, version, err := getResponse("node", "--version")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return &pulumirpc.AboutResponse{
|
|
|
|
Executable: node,
|
|
|
|
Version: version,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// The shape of a `yarn list --json`'s output.
|
|
|
|
type yarnLock struct {
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data yarnLockData `json:"data"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type yarnLockData struct {
|
|
|
|
Type string `json:"type"`
|
|
|
|
Trees []yarnLockTree `json:"trees"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type yarnLockTree struct {
|
|
|
|
Name string `json:"name"`
|
|
|
|
Children []yarnLockTree `json:"children"`
|
|
|
|
}
|
|
|
|
|
2024-02-06 12:38:44 +00:00
|
|
|
func parseYarnLockFile(programDirectory, path string) ([]*pulumirpc.DependencyInfo, error) {
|
2022-08-15 13:55:04 +00:00
|
|
|
ex, err := executable.FindExecutable("yarn")
|
|
|
|
if err != nil {
|
2022-10-09 14:58:33 +00:00
|
|
|
return nil, fmt.Errorf("found %s but no yarn executable: %w", path, err)
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
cmdArgs := []string{"list", "--json"}
|
|
|
|
cmd := exec.Command(ex, cmdArgs...)
|
2024-02-06 12:38:44 +00:00
|
|
|
cmd.Dir = programDirectory
|
2022-08-15 13:55:04 +00:00
|
|
|
out, err := cmd.Output()
|
|
|
|
if err != nil {
|
2022-10-09 14:58:33 +00:00
|
|
|
return nil, fmt.Errorf("failed to run \"%s %s\": %w", ex, strings.Join(cmdArgs, " "), err)
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var lock yarnLock
|
|
|
|
if err = json.Unmarshal(out, &lock); err != nil {
|
2022-10-09 14:58:33 +00:00
|
|
|
return nil, fmt.Errorf("failed to parse\"%s %s\": %w", ex, strings.Join(cmdArgs, " "), err)
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
leafs := lock.Data.Trees
|
|
|
|
|
|
|
|
result := make([]*pulumirpc.DependencyInfo, len(leafs))
|
|
|
|
|
|
|
|
// Has the form name@version
|
|
|
|
splitName := func(index int, nameVersion string) (string, string, error) {
|
|
|
|
if nameVersion == "" {
|
2022-10-09 14:58:33 +00:00
|
|
|
return "", "", fmt.Errorf("expected \"name\" in dependency %d", index)
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
split := strings.LastIndex(nameVersion, "@")
|
|
|
|
if split == -1 {
|
2022-10-09 14:58:33 +00:00
|
|
|
return "", "", fmt.Errorf("failed to parse name and version from %s", nameVersion)
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
return nameVersion[:split], nameVersion[split+1:], nil
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, v := range leafs {
|
|
|
|
name, version, err := splitName(i, v.Name)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
result[i] = &pulumirpc.DependencyInfo{
|
|
|
|
Name: name,
|
|
|
|
Version: version,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return result, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Describes the shape of `npm ls --json --depth=0`'s output.
|
|
|
|
type npmFile struct {
|
|
|
|
Name string `json:"name"`
|
|
|
|
LockFileVersion int `json:"lockfileVersion"`
|
|
|
|
Requires bool `json:"requires"`
|
|
|
|
Dependencies map[string]npmPackage `json:"dependencies"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// A package in npmFile.
|
|
|
|
type npmPackage struct {
|
|
|
|
Version string `json:"version"`
|
|
|
|
Resolved string `json:"resolved"`
|
|
|
|
}
|
|
|
|
|
2024-02-06 12:38:44 +00:00
|
|
|
func parseNpmLockFile(programDirectory, path string) ([]*pulumirpc.DependencyInfo, error) {
|
2022-08-15 13:55:04 +00:00
|
|
|
ex, err := executable.FindExecutable("npm")
|
|
|
|
if err != nil {
|
2022-10-09 14:58:33 +00:00
|
|
|
return nil, fmt.Errorf("found %s but not npm: %w", path, err)
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
cmdArgs := []string{"ls", "--json", "--depth=0"}
|
|
|
|
cmd := exec.Command(ex, cmdArgs...)
|
2024-02-06 12:38:44 +00:00
|
|
|
cmd.Dir = programDirectory
|
2022-08-15 13:55:04 +00:00
|
|
|
out, err := cmd.Output()
|
|
|
|
if err != nil {
|
2022-10-09 14:58:33 +00:00
|
|
|
return nil, fmt.Errorf(`failed to run "%s %s": %w`, ex, strings.Join(cmdArgs, " "), err)
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
file := npmFile{}
|
|
|
|
if err = json.Unmarshal(out, &file); err != nil {
|
2022-10-09 14:58:33 +00:00
|
|
|
return nil, fmt.Errorf(`failed to parse \"%s %s": %w`, ex, strings.Join(cmdArgs, " "), err)
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
result := make([]*pulumirpc.DependencyInfo, len(file.Dependencies))
|
|
|
|
var i int
|
|
|
|
for k, v := range file.Dependencies {
|
|
|
|
result[i] = &pulumirpc.DependencyInfo{
|
|
|
|
Name: k,
|
|
|
|
Version: v.Version,
|
|
|
|
}
|
|
|
|
i++
|
|
|
|
}
|
|
|
|
return result, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Intersect a list of packages with the contents of `package.json`. Returns
|
|
|
|
// only packages that appear in both sets. `path` is used only for error handling.
|
|
|
|
func crossCheckPackageJSONFile(path string, file []byte,
|
2023-03-03 16:36:39 +00:00
|
|
|
packages []*pulumirpc.DependencyInfo,
|
|
|
|
) ([]*pulumirpc.DependencyInfo, error) {
|
2022-08-15 13:55:04 +00:00
|
|
|
var body packageJSON
|
|
|
|
if err := json.Unmarshal(file, &body); err != nil {
|
2022-10-09 14:58:33 +00:00
|
|
|
return nil, fmt.Errorf("could not parse %s: %w", path, err)
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
dependencies := make(map[string]string)
|
|
|
|
for k, v := range body.Dependencies {
|
|
|
|
dependencies[k] = v
|
|
|
|
}
|
|
|
|
for k, v := range body.DevDependencies {
|
|
|
|
dependencies[k] = v
|
|
|
|
}
|
|
|
|
|
|
|
|
// There should be 1 (& only 1) instantiated dependency for each
|
|
|
|
// dependency in package.json. We do this because we want to get the
|
|
|
|
// actual version (not the range) that exists in lock files.
|
|
|
|
result := make([]*pulumirpc.DependencyInfo, len(dependencies))
|
|
|
|
i := 0
|
|
|
|
for _, v := range packages {
|
|
|
|
if _, exists := dependencies[v.Name]; exists {
|
|
|
|
result[i] = v
|
|
|
|
// Some direct dependencies are also transitive dependencies. We
|
|
|
|
// only want to grab them once.
|
|
|
|
delete(dependencies, v.Name)
|
|
|
|
i++
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return result, nil
|
2022-07-25 11:35:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (host *nodeLanguageHost) GetProgramDependencies(
|
2023-03-03 16:36:39 +00:00
|
|
|
ctx context.Context, req *pulumirpc.GetProgramDependenciesRequest,
|
|
|
|
) (*pulumirpc.GetProgramDependenciesResponse, error) {
|
2022-08-15 13:55:04 +00:00
|
|
|
// We get the node dependencies. This requires either a yarn.lock file and the
|
|
|
|
// yarn executable, a package-lock.json file and the npm executable. If
|
|
|
|
// transitive is false, we also need the package.json file.
|
|
|
|
//
|
|
|
|
// If we find a yarn.lock file, we assume that yarn is used.
|
|
|
|
// Only then do we look for a package-lock.json file.
|
|
|
|
|
|
|
|
// Neither "yarn list" or "npm ls" can describe what packages are required
|
|
|
|
//
|
|
|
|
// (direct dependencies). Only what packages they have installed (transitive
|
|
|
|
// dependencies). This means that to accurately report only direct
|
|
|
|
// dependencies, we need to also parse "package.json" and intersect it with
|
|
|
|
// reported dependencies.
|
|
|
|
var err error
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
yarnFile := filepath.Join(req.Info.ProgramDirectory, "yarn.lock")
|
|
|
|
npmFile := filepath.Join(req.Info.ProgramDirectory, "package-lock.json")
|
|
|
|
packageFile := filepath.Join(req.Info.ProgramDirectory, "package.json")
|
2022-08-15 13:55:04 +00:00
|
|
|
var result []*pulumirpc.DependencyInfo
|
|
|
|
|
|
|
|
if _, err = os.Stat(yarnFile); err == nil {
|
2024-02-06 12:38:44 +00:00
|
|
|
result, err = parseYarnLockFile(req.Info.ProgramDirectory, yarnFile)
|
2022-08-15 13:55:04 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
} else if _, err = os.Stat(npmFile); err == nil {
|
2024-02-06 12:38:44 +00:00
|
|
|
result, err = parseNpmLockFile(req.Info.ProgramDirectory, npmFile)
|
2022-08-15 13:55:04 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
} else if os.IsNotExist(err) {
|
2022-10-09 14:58:33 +00:00
|
|
|
return nil, fmt.Errorf("could not find either %s or %s", yarnFile, npmFile)
|
2022-08-15 13:55:04 +00:00
|
|
|
} else {
|
2022-10-09 14:58:33 +00:00
|
|
|
return nil, fmt.Errorf("could not get node dependency data: %w", err)
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
if !req.TransitiveDependencies {
|
2023-01-06 22:39:16 +00:00
|
|
|
file, err := os.ReadFile(packageFile)
|
2022-08-15 13:55:04 +00:00
|
|
|
if os.IsNotExist(err) {
|
2022-10-09 14:58:33 +00:00
|
|
|
return nil, fmt.Errorf("could not find %s. "+
|
2022-08-15 13:55:04 +00:00
|
|
|
"Please include this in your report and run "+
|
|
|
|
`pulumi about --transitive" to get a list of used packages`,
|
|
|
|
packageFile)
|
|
|
|
} else if err != nil {
|
2022-10-09 14:58:33 +00:00
|
|
|
return nil, fmt.Errorf("could not read %s: %w", packageFile, err)
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
result, err = crossCheckPackageJSONFile(packageFile, file, result)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return &pulumirpc.GetProgramDependenciesResponse{
|
|
|
|
Dependencies: result,
|
|
|
|
}, nil
|
2022-07-25 11:35:16 +00:00
|
|
|
}
|
2022-10-04 08:58:01 +00:00
|
|
|
|
|
|
|
func (host *nodeLanguageHost) RunPlugin(
|
2023-03-03 16:36:39 +00:00
|
|
|
req *pulumirpc.RunPluginRequest, server pulumirpc.LanguageRuntime_RunPluginServer,
|
|
|
|
) error {
|
2022-10-04 08:58:01 +00:00
|
|
|
return errors.New("not supported")
|
|
|
|
}
|
2023-05-12 13:38:36 +00:00
|
|
|
|
|
|
|
func (host *nodeLanguageHost) GenerateProject(
|
|
|
|
ctx context.Context, req *pulumirpc.GenerateProjectRequest,
|
|
|
|
) (*pulumirpc.GenerateProjectResponse, error) {
|
2023-07-27 09:27:07 +00:00
|
|
|
loader, err := schema.NewLoaderClient(req.LoaderTarget)
|
2023-05-12 13:38:36 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2023-08-31 16:35:21 +00:00
|
|
|
var extraOptions []pcl.BindOption
|
2023-06-23 14:19:02 +00:00
|
|
|
if !req.Strict {
|
2023-07-13 13:16:06 +00:00
|
|
|
extraOptions = append(extraOptions, pcl.NonStrictBindOptions()...)
|
2023-06-23 00:42:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// for nodejs, prefer output-versioned invokes
|
|
|
|
extraOptions = append(extraOptions, pcl.PreferOutputVersionedInvokes)
|
|
|
|
|
|
|
|
program, diags, err := pcl.BindDirectory(req.SourceDirectory, loader, extraOptions...)
|
2023-05-12 13:38:36 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-05-26 10:32:19 +00:00
|
|
|
if diags.HasErrors() {
|
2023-08-31 16:35:21 +00:00
|
|
|
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
|
2023-05-26 10:32:19 +00:00
|
|
|
|
|
|
|
return &pulumirpc.GenerateProjectResponse{
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
}, nil
|
2023-05-12 13:38:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var project workspace.Project
|
|
|
|
if err := json.Unmarshal([]byte(req.Project), &project); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2024-03-28 15:44:25 +00:00
|
|
|
err = codegen.GenerateProject(req.TargetDirectory, project, program, req.LocalDependencies, host.forceTsc)
|
2023-05-12 13:38:36 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2023-08-31 16:35:21 +00:00
|
|
|
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
|
2023-05-26 10:32:19 +00:00
|
|
|
|
|
|
|
return &pulumirpc.GenerateProjectResponse{
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
}, nil
|
2023-05-12 13:38:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (host *nodeLanguageHost) GenerateProgram(
|
|
|
|
ctx context.Context, req *pulumirpc.GenerateProgramRequest,
|
|
|
|
) (*pulumirpc.GenerateProgramResponse, error) {
|
2023-07-27 09:27:07 +00:00
|
|
|
loader, err := schema.NewLoaderClient(req.LoaderTarget)
|
2023-05-12 13:38:36 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
parser := hclsyntax.NewParser()
|
|
|
|
// Load all .pp files in the directory
|
|
|
|
for path, contents := range req.Source {
|
|
|
|
err = parser.ParseFile(strings.NewReader(contents), path)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
diags := parser.Diagnostics
|
|
|
|
if diags.HasErrors() {
|
|
|
|
return nil, diags
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
[cli/import] Fix undefined variable errors in code generation when imported resources use a parent or provider (#16786)
Fixes #15410 Fixes #13339
## Problem Context
When using `pulumi import` we generate code snippets for the resources
that were imported. Sometimes the user specifies `--parent
parentName=URN` or `--provider providerName=URN` which tweak the parent
or provider that the imported resources uses. When using `--parent` or
`--provider` the generated code emits a resource option `parent =
parentName` (in case of using `--parent`) where `parentName` is an
unbound variable.
Usually unbound variables would result in a _bind_ error such as `error:
undefined variable parentName` when type-checking the program however in
the import code generation we specify the bind option
`pcl.AllowMissingVariables` which turns that unbound variable errors
into warnings and code generation can continue to emit code.
This is all good and works as expected. However in the issues linked
above, we do get an _error_ for unbound variables in generated code even
though we specified `AllowMissingVariables`.
The problem as it turns out is when we are trying to generate code via
dynamically loaded `LangaugeRuntime` plugins. Specifically for NodeJS
and Python, we load `pulumi-language-nodejs` or `pulumi-language-python`
and call `GenerateProgram` to get the generated program. That function
`GenerateProgram` takes the text _SOURCE_ of the a bound program (one
that was bound using option `AllowMissingVariables`) and re-binds again
inside the implementation of the language plugin. The second time we
bind the program, we don't pass it the option `AllowMissingVariables`
and so it fails with `unboud variable` error.
I've verified that the issue above don't repro when doing an import for
dotnet (probably same for java/yaml) because we use the statically
linked function `codegen/{lang}/gen_program.go -> GenerateProgram`
## Solution
The problem can be solved by propagating the bind options from the CLI
to the language hosts during import so that they know how to bind the
program. I've extended the gRPC interface in `GenerateProgramRequest`
with a property `Strict` which follows the same logic from `pulumi
convert --strict` and made it such that the import command sends
`strict=false` to the language plugins when doing `GenerateProgram`.
This is consistent with `GenerateProject` that uses the same flag. When
`strict=false` we use `pcl.NonStrictBindOptions()` which includes
`AllowMissingVariables` .
## Repro
Once can test the before and after behaviour by running `pulumi up
--yes` on the following TypeScript program:
```ts
import * as pulumi from "@pulumi/pulumi";
import * as random from "@pulumi/random";
export class MyComponent extends pulumi.ComponentResource {
public readonly randomPetId: pulumi.Output<string>;
constructor(name: string, opts?: pulumi.ComponentResourceOptions) {
super("example:index:MyComponent", name, {}, opts);
const randomPet = new random.RandomPet("randomPet", {}, {
parent: this
});
this.randomPetId = randomPet.id;
this.registerOutputs({
randomPetId: randomPet.id,
});
}
}
const example = new MyComponent("example");
export const randomPetId = example.randomPetId;
```
Then running `pulumi import -f import.json` where `import.json` contains
a resource to be imported under the created component (stack=`dev`,
project=`importerrors`)
```ts
{
"nameTable": {
"parentComponent": "urn:pulumi:dev::importerrors::example:index:MyComponent::example"
},
"resources": [
{
"type": "random:index/randomPassword:RandomPassword",
"name": "randomPassword",
"id": "supersecret",
"parent": "parentComponent"
}
]
}
```
Running this locally I get the following generated code (which
previously failed to generate)
```ts
import * as pulumi from "@pulumi/pulumi";
import * as random from "@pulumi/random";
const randomPassword = new random.RandomPassword("randomPassword", {
length: 11,
lower: true,
number: true,
numeric: true,
special: true,
upper: true,
}, {
parent: parentComponent,
});
```
2024-07-25 13:53:44 +00:00
|
|
|
bindOptions := []pcl.BindOption{
|
2023-06-23 00:42:18 +00:00
|
|
|
pcl.Loader(loader),
|
[cli/import] Fix undefined variable errors in code generation when imported resources use a parent or provider (#16786)
Fixes #15410 Fixes #13339
## Problem Context
When using `pulumi import` we generate code snippets for the resources
that were imported. Sometimes the user specifies `--parent
parentName=URN` or `--provider providerName=URN` which tweak the parent
or provider that the imported resources uses. When using `--parent` or
`--provider` the generated code emits a resource option `parent =
parentName` (in case of using `--parent`) where `parentName` is an
unbound variable.
Usually unbound variables would result in a _bind_ error such as `error:
undefined variable parentName` when type-checking the program however in
the import code generation we specify the bind option
`pcl.AllowMissingVariables` which turns that unbound variable errors
into warnings and code generation can continue to emit code.
This is all good and works as expected. However in the issues linked
above, we do get an _error_ for unbound variables in generated code even
though we specified `AllowMissingVariables`.
The problem as it turns out is when we are trying to generate code via
dynamically loaded `LangaugeRuntime` plugins. Specifically for NodeJS
and Python, we load `pulumi-language-nodejs` or `pulumi-language-python`
and call `GenerateProgram` to get the generated program. That function
`GenerateProgram` takes the text _SOURCE_ of the a bound program (one
that was bound using option `AllowMissingVariables`) and re-binds again
inside the implementation of the language plugin. The second time we
bind the program, we don't pass it the option `AllowMissingVariables`
and so it fails with `unboud variable` error.
I've verified that the issue above don't repro when doing an import for
dotnet (probably same for java/yaml) because we use the statically
linked function `codegen/{lang}/gen_program.go -> GenerateProgram`
## Solution
The problem can be solved by propagating the bind options from the CLI
to the language hosts during import so that they know how to bind the
program. I've extended the gRPC interface in `GenerateProgramRequest`
with a property `Strict` which follows the same logic from `pulumi
convert --strict` and made it such that the import command sends
`strict=false` to the language plugins when doing `GenerateProgram`.
This is consistent with `GenerateProject` that uses the same flag. When
`strict=false` we use `pcl.NonStrictBindOptions()` which includes
`AllowMissingVariables` .
## Repro
Once can test the before and after behaviour by running `pulumi up
--yes` on the following TypeScript program:
```ts
import * as pulumi from "@pulumi/pulumi";
import * as random from "@pulumi/random";
export class MyComponent extends pulumi.ComponentResource {
public readonly randomPetId: pulumi.Output<string>;
constructor(name: string, opts?: pulumi.ComponentResourceOptions) {
super("example:index:MyComponent", name, {}, opts);
const randomPet = new random.RandomPet("randomPet", {}, {
parent: this
});
this.randomPetId = randomPet.id;
this.registerOutputs({
randomPetId: randomPet.id,
});
}
}
const example = new MyComponent("example");
export const randomPetId = example.randomPetId;
```
Then running `pulumi import -f import.json` where `import.json` contains
a resource to be imported under the created component (stack=`dev`,
project=`importerrors`)
```ts
{
"nameTable": {
"parentComponent": "urn:pulumi:dev::importerrors::example:index:MyComponent::example"
},
"resources": [
{
"type": "random:index/randomPassword:RandomPassword",
"name": "randomPassword",
"id": "supersecret",
"parent": "parentComponent"
}
]
}
```
Running this locally I get the following generated code (which
previously failed to generate)
```ts
import * as pulumi from "@pulumi/pulumi";
import * as random from "@pulumi/random";
const randomPassword = new random.RandomPassword("randomPassword", {
length: 11,
lower: true,
number: true,
numeric: true,
special: true,
upper: true,
}, {
parent: parentComponent,
});
```
2024-07-25 13:53:44 +00:00
|
|
|
// for nodejs, prefer output-versioned invokes
|
|
|
|
pcl.PreferOutputVersionedInvokes,
|
|
|
|
}
|
|
|
|
|
|
|
|
if !req.Strict {
|
|
|
|
bindOptions = append(bindOptions, pcl.NonStrictBindOptions()...)
|
|
|
|
}
|
|
|
|
|
|
|
|
program, diags, err := pcl.BindProgram(parser.Files, bindOptions...)
|
2023-05-12 13:38:36 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-06-07 17:17:59 +00:00
|
|
|
|
2023-08-31 16:35:21 +00:00
|
|
|
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
|
|
|
|
if diags.HasErrors() {
|
2023-06-07 17:17:59 +00:00
|
|
|
return &pulumirpc.GenerateProgramResponse{
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
if program == nil {
|
2024-04-19 06:20:33 +00:00
|
|
|
return nil, errors.New("internal error program was nil")
|
2023-05-12 13:38:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
files, diags, err := codegen.GenerateProgram(program)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-08-31 16:35:21 +00:00
|
|
|
rpcDiagnostics = append(rpcDiagnostics, plugin.HclDiagnosticsToRPCDiagnostics(diags)...)
|
2023-05-12 13:38:36 +00:00
|
|
|
|
|
|
|
return &pulumirpc.GenerateProgramResponse{
|
|
|
|
Source: files,
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (host *nodeLanguageHost) GeneratePackage(
|
|
|
|
ctx context.Context, req *pulumirpc.GeneratePackageRequest,
|
|
|
|
) (*pulumirpc.GeneratePackageResponse, error) {
|
2023-07-27 09:27:07 +00:00
|
|
|
loader, err := schema.NewLoaderClient(req.LoaderTarget)
|
2023-05-12 13:38:36 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
var spec schema.PackageSpec
|
|
|
|
err = json.Unmarshal([]byte(req.Schema), &spec)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
pkg, diags, err := schema.BindSpec(spec, loader)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-12-05 17:47:52 +00:00
|
|
|
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
|
2023-05-12 13:38:36 +00:00
|
|
|
if diags.HasErrors() {
|
2023-12-05 17:47:52 +00:00
|
|
|
return &pulumirpc.GeneratePackageResponse{
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
}, nil
|
2023-05-12 13:38:36 +00:00
|
|
|
}
|
2024-03-26 13:10:34 +00:00
|
|
|
|
|
|
|
files, err := codegen.GeneratePackage("pulumi-language-nodejs", pkg, req.ExtraFiles, req.LocalDependencies)
|
2023-05-12 13:38:36 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
for filename, data := range files {
|
2023-08-31 16:35:21 +00:00
|
|
|
outPath := filepath.Join(req.Directory, filename)
|
2023-05-12 13:38:36 +00:00
|
|
|
|
|
|
|
err := os.MkdirAll(filepath.Dir(outPath), 0o700)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("could not create output directory %s: %w", filepath.Dir(filename), err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = os.WriteFile(outPath, data, 0o600)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("could not write output file %s: %w", filename, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-05 17:47:52 +00:00
|
|
|
return &pulumirpc.GeneratePackageResponse{
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
}, nil
|
2023-05-12 13:38:36 +00:00
|
|
|
}
|
2023-07-27 21:39:36 +00:00
|
|
|
|
|
|
|
func readPackageJSON(packageJSONPath string) (map[string]interface{}, error) {
|
|
|
|
packageJSONData, err := os.ReadFile(packageJSONPath)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("read package.json: %w", err)
|
|
|
|
}
|
|
|
|
var packageJSON map[string]interface{}
|
|
|
|
err = json.Unmarshal(packageJSONData, &packageJSON)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("unmarshal package.json: %w", err)
|
|
|
|
}
|
|
|
|
return packageJSON, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (host *nodeLanguageHost) Pack(ctx context.Context, req *pulumirpc.PackRequest) (*pulumirpc.PackResponse, error) {
|
|
|
|
// Verify npm exists and is set up: npm, user login
|
|
|
|
npm, err := executable.FindExecutable("npm")
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("find npm: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Annoyingly the engine will call Pack for the core SDK which is not setup in at all the same way as the
|
|
|
|
// generated sdks, so we have to detect that and do a big branch to pack it totally differently.
|
|
|
|
packageJSON, err := readPackageJSON(filepath.Join(req.PackageDirectory, "package.json"))
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// TODO: We're just going to write to stderr for now, but we should probably have a way to return this
|
|
|
|
// directly to the engine hosting us. That's a bit awkward because we need the subprocesses to write to
|
|
|
|
// this as well, and the host side of this pipe might be a tty and if it is we want the subprocesses to
|
|
|
|
// think they're connected to a tty as well. We've solved this problem before in two slightly different
|
|
|
|
// ways for InstallDependencies and RunPlugin, it would be good to come up with a clean way to do this for
|
|
|
|
// all these cases.
|
|
|
|
writeString := func(s string) error {
|
|
|
|
_, err = os.Stderr.Write([]byte(s))
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if packageJSON["name"] == "@pulumi/pulumi" {
|
|
|
|
// This is pretty much a copy of the makefiles build_package. Short term we should see about changing
|
|
|
|
// the makefile to just build the nodejs plugin first and then simply invoke "pulumi package
|
|
|
|
// pack-sdk". Long term we should try and unify the style of the code sdk with that of generated sdks
|
|
|
|
// so we don't need this special case.
|
|
|
|
|
|
|
|
yarn, err := executable.FindExecutable("yarn")
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("find yarn: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = writeString("$ yarn install --frozen-lockfile\n")
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("write to output: %w", err)
|
|
|
|
}
|
|
|
|
yarnInstallCmd := exec.Command(yarn, "install", "--frozen-lockfile")
|
|
|
|
yarnInstallCmd.Dir = req.PackageDirectory
|
2024-07-23 16:10:43 +00:00
|
|
|
if err := runWithOutput(yarnInstallCmd, os.Stdout, os.Stderr); err != nil {
|
2023-07-27 21:39:36 +00:00
|
|
|
return nil, fmt.Errorf("yarn install: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = writeString("$ yarn run tsc\n")
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("write to output: %w", err)
|
|
|
|
}
|
|
|
|
yarnTscCmd := exec.Command(yarn, "run", "tsc")
|
|
|
|
yarnTscCmd.Dir = req.PackageDirectory
|
2024-07-23 16:10:43 +00:00
|
|
|
if err := runWithOutput(yarnTscCmd, os.Stdout, os.Stderr); err != nil {
|
2023-07-27 21:39:36 +00:00
|
|
|
return nil, fmt.Errorf("yarn run tsc: %w", err)
|
|
|
|
}
|
Add matrix testing (#13705)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
Adds the first pass of matrix testing.
Matrix testing allows us to define tests once in pulumi/pulumi via PCL
and then run those tests against each language plugin to verify code
generation and runtime correctness.
Rather than packing matrix tests and all the associated data and
machinery into the CLI itself we define a new Go package at
cmd/pulumi-test-lanaguage. This depends on pkg and runs the deployment
engine in a unique way for matrix tests but it is running the proper
deployment engine with a proper backend (always filestate, using $TEMP).
Currently only NodeJS is hooked up to run these tests, and all the code
for that currently lives in
sdk/nodejs/cmd/pulumi-language-nodejs/language_test.go. I expect we'll
move that helper code to sdk/go/common and use it in each language
plugin to run the tests in the same way.
This first pass includes 3 simple tests:
* l1-empty that runs an empty PCL file and checks just a stack is
created
* l1-output-bool that runs a PCL program that returns two stack outputs
of `true` and `false
* l2-resource-simple that runs a PCL program creating a simple resource
with a single bool property
These tests are themselves tested with a mock language runtime. This
verifies the behavior of the matrix test framework for both correct and
incorrect language hosts (that is some the mock language runtimes
purposefully cause errors or compute the wrong result).
There are a number of things missing from from the core framework still,
but I feel don't block getting this first pass merged and starting to be
used.
1. The tests can not currently run in parallel. That is calling
RunLanguageTest in parallel will break things. This is due to two
separate problems. Firstly is that the SDK snapshot's are not safe to
write in parallel (when PULUMI_ACCEPT is true), this should be fairly
easy to fix by doing a write to dst-{random} and them atomic move to
dst. Secondly is that the deployment engine itself has mutable global
state, short term we should probably just lock around that part
RunLanguageTest, long term it would be good to clean that up.
2. We need a way to verify "preview" behavior, I think this is probably
just a variation of the tests that would call `stack.Preview` and not
pass a snapshot to `assert`.
3. stdout, stderr and log messages are returned in bulk at the end of
the test. Plus there are a couple of calls to the language runtime that
don't correctly thread stdout/stderr to use and so default to the
process `os.Stdout/Stderr`. stdout/stderr streaming shows up in a load
of other places as well so I'm thinking of a clean way to handle all of
them together. Log message streaming we can probably do by just turning
RunLanguageTest to a streaming grpc call.
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [x] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
---------
Co-authored-by: Abhinav Gupta <abhinav@pulumi.com>
2023-09-13 15:17:46 +00:00
|
|
|
|
2024-04-10 15:26:37 +00:00
|
|
|
// "tsc" doesn't copy in the "proto" and "vendor" directories.
|
Add matrix testing (#13705)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
Adds the first pass of matrix testing.
Matrix testing allows us to define tests once in pulumi/pulumi via PCL
and then run those tests against each language plugin to verify code
generation and runtime correctness.
Rather than packing matrix tests and all the associated data and
machinery into the CLI itself we define a new Go package at
cmd/pulumi-test-lanaguage. This depends on pkg and runs the deployment
engine in a unique way for matrix tests but it is running the proper
deployment engine with a proper backend (always filestate, using $TEMP).
Currently only NodeJS is hooked up to run these tests, and all the code
for that currently lives in
sdk/nodejs/cmd/pulumi-language-nodejs/language_test.go. I expect we'll
move that helper code to sdk/go/common and use it in each language
plugin to run the tests in the same way.
This first pass includes 3 simple tests:
* l1-empty that runs an empty PCL file and checks just a stack is
created
* l1-output-bool that runs a PCL program that returns two stack outputs
of `true` and `false
* l2-resource-simple that runs a PCL program creating a simple resource
with a single bool property
These tests are themselves tested with a mock language runtime. This
verifies the behavior of the matrix test framework for both correct and
incorrect language hosts (that is some the mock language runtimes
purposefully cause errors or compute the wrong result).
There are a number of things missing from from the core framework still,
but I feel don't block getting this first pass merged and starting to be
used.
1. The tests can not currently run in parallel. That is calling
RunLanguageTest in parallel will break things. This is due to two
separate problems. Firstly is that the SDK snapshot's are not safe to
write in parallel (when PULUMI_ACCEPT is true), this should be fairly
easy to fix by doing a write to dst-{random} and them atomic move to
dst. Secondly is that the deployment engine itself has mutable global
state, short term we should probably just lock around that part
RunLanguageTest, long term it would be good to clean that up.
2. We need a way to verify "preview" behavior, I think this is probably
just a variation of the tests that would call `stack.Preview` and not
pass a snapshot to `assert`.
3. stdout, stderr and log messages are returned in bulk at the end of
the test. Plus there are a couple of calls to the language runtime that
don't correctly thread stdout/stderr to use and so default to the
process `os.Stdout/Stderr`. stdout/stderr streaming shows up in a load
of other places as well so I'm thinking of a clean way to handle all of
them together. Log message streaming we can probably do by just turning
RunLanguageTest to a streaming grpc call.
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [x] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
---------
Co-authored-by: Abhinav Gupta <abhinav@pulumi.com>
2023-09-13 15:17:46 +00:00
|
|
|
err = fsutil.CopyFile(
|
|
|
|
filepath.Join(req.PackageDirectory, "bin", "proto"),
|
|
|
|
filepath.Join(req.PackageDirectory, "proto"),
|
|
|
|
nil)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("copy proto: %w", err)
|
|
|
|
}
|
2024-04-10 15:26:37 +00:00
|
|
|
err = fsutil.CopyFile(
|
|
|
|
filepath.Join(req.PackageDirectory, "bin", "vendor"),
|
|
|
|
filepath.Join(req.PackageDirectory, "vendor"),
|
|
|
|
nil)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("copy vendor: %w", err)
|
|
|
|
}
|
Add matrix testing (#13705)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
Adds the first pass of matrix testing.
Matrix testing allows us to define tests once in pulumi/pulumi via PCL
and then run those tests against each language plugin to verify code
generation and runtime correctness.
Rather than packing matrix tests and all the associated data and
machinery into the CLI itself we define a new Go package at
cmd/pulumi-test-lanaguage. This depends on pkg and runs the deployment
engine in a unique way for matrix tests but it is running the proper
deployment engine with a proper backend (always filestate, using $TEMP).
Currently only NodeJS is hooked up to run these tests, and all the code
for that currently lives in
sdk/nodejs/cmd/pulumi-language-nodejs/language_test.go. I expect we'll
move that helper code to sdk/go/common and use it in each language
plugin to run the tests in the same way.
This first pass includes 3 simple tests:
* l1-empty that runs an empty PCL file and checks just a stack is
created
* l1-output-bool that runs a PCL program that returns two stack outputs
of `true` and `false
* l2-resource-simple that runs a PCL program creating a simple resource
with a single bool property
These tests are themselves tested with a mock language runtime. This
verifies the behavior of the matrix test framework for both correct and
incorrect language hosts (that is some the mock language runtimes
purposefully cause errors or compute the wrong result).
There are a number of things missing from from the core framework still,
but I feel don't block getting this first pass merged and starting to be
used.
1. The tests can not currently run in parallel. That is calling
RunLanguageTest in parallel will break things. This is due to two
separate problems. Firstly is that the SDK snapshot's are not safe to
write in parallel (when PULUMI_ACCEPT is true), this should be fairly
easy to fix by doing a write to dst-{random} and them atomic move to
dst. Secondly is that the deployment engine itself has mutable global
state, short term we should probably just lock around that part
RunLanguageTest, long term it would be good to clean that up.
2. We need a way to verify "preview" behavior, I think this is probably
just a variation of the tests that would call `stack.Preview` and not
pass a snapshot to `assert`.
3. stdout, stderr and log messages are returned in bulk at the end of
the test. Plus there are a couple of calls to the language runtime that
don't correctly thread stdout/stderr to use and so default to the
process `os.Stdout/Stderr`. stdout/stderr streaming shows up in a load
of other places as well so I'm thinking of a clean way to handle all of
them together. Log message streaming we can probably do by just turning
RunLanguageTest to a streaming grpc call.
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [x] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
---------
Co-authored-by: Abhinav Gupta <abhinav@pulumi.com>
2023-09-13 15:17:46 +00:00
|
|
|
|
2023-07-27 21:39:36 +00:00
|
|
|
} else {
|
|
|
|
// Before we can build the package we need to install it's dependencies.
|
|
|
|
err = writeString("$ npm install\n")
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("write to output: %w", err)
|
|
|
|
}
|
|
|
|
npmInstallCmd := exec.Command(npm, "install")
|
|
|
|
npmInstallCmd.Dir = req.PackageDirectory
|
2024-07-23 16:10:43 +00:00
|
|
|
if err := runWithOutput(npmInstallCmd, os.Stdout, os.Stderr); err != nil {
|
2023-07-27 21:39:36 +00:00
|
|
|
return nil, fmt.Errorf("npm install: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Pulumi SDKs always define a build command that will run tsc writing to a bin directory.
|
|
|
|
// So we can run that, then edit the package.json in that directory, and then pack it.
|
|
|
|
err = writeString("$ npm run build\n")
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("write to output: %w", err)
|
|
|
|
}
|
|
|
|
npmBuildCmd := exec.Command(npm, "run", "build")
|
|
|
|
npmBuildCmd.Dir = req.PackageDirectory
|
2024-07-23 16:10:43 +00:00
|
|
|
if err := runWithOutput(npmBuildCmd, os.Stdout, os.Stderr); err != nil {
|
2023-07-27 21:39:36 +00:00
|
|
|
return nil, fmt.Errorf("npm run build: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// "build" in SDKs isn't setup to copy the package.json to ./bin/
|
|
|
|
err = fsutil.CopyFile(
|
|
|
|
filepath.Join(req.PackageDirectory, "bin", "package.json"),
|
|
|
|
filepath.Join(req.PackageDirectory, "package.json"),
|
|
|
|
nil)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("copy package.json: %w", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
err = writeString("$ npm pack\n")
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("write to output: %w", err)
|
|
|
|
}
|
|
|
|
var stdoutBuffer bytes.Buffer
|
|
|
|
npmPackCmd := exec.Command(npm,
|
|
|
|
"pack",
|
|
|
|
filepath.Join(req.PackageDirectory, "bin"),
|
|
|
|
"--pack-destination", req.DestinationDirectory)
|
|
|
|
npmPackCmd.Stdout = &stdoutBuffer
|
2024-07-23 16:10:43 +00:00
|
|
|
npmPackCmd.Stderr = struct{ io.Writer }{os.Stderr}
|
2023-07-27 21:39:36 +00:00
|
|
|
err = npmPackCmd.Run()
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("npm pack: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
artifactName := strings.TrimSpace(stdoutBuffer.String())
|
|
|
|
|
|
|
|
return &pulumirpc.PackResponse{
|
|
|
|
ArtifactPath: filepath.Join(req.DestinationDirectory, artifactName),
|
|
|
|
}, nil
|
|
|
|
}
|
2024-06-27 18:34:31 +00:00
|
|
|
|
2024-07-23 16:10:43 +00:00
|
|
|
// Nodejs sometimes sets stdout/stderr to non-blocking mode. When a nodejs subprocess is directly
|
|
|
|
// handed the go process's stdout/stderr file descriptors, nodejs's non-blocking configuration goes
|
|
|
|
// unnoticed by go, and a write from go can result in an error `write /dev/stdout: resource
|
|
|
|
// temporarily unavailable`.
|
|
|
|
//
|
|
|
|
// The solution to this is to not provide nodejs with the go process's stdout/stderr file
|
|
|
|
// descriptors, and instead proxy the writes through something else.
|
|
|
|
// In https://github.com/pulumi/pulumi/pull/16504 we used Cmd.StdoutPipe/StderrPipe for this.
|
|
|
|
// However this introduced a potential bug, as it is not safe to use these specific pipes along
|
|
|
|
// with Cmd.Run. The issue is that these pipes will be closed as soon as the process exits, which
|
|
|
|
// can lead to missing data when stdout/stderr are slow, or worse an error due to an attempted read
|
|
|
|
// from a closed pipe. (Creating and closing our own os.Pipes for this would be fine.)
|
|
|
|
//
|
|
|
|
// A simpler workaround is to wrap stdout/stderr in an io.Writer. As with the pipes, this makes
|
|
|
|
// exec.Cmd use a copying goroutine to shuffle the data from the subprocess to stdout/stderr.
|
|
|
|
//
|
|
|
|
// Cmd.Run will wait for all the data to be copied before returning, ensuring we do not miss any data.
|
|
|
|
//
|
|
|
|
// Non-blocking issue: https://github.com/golang/go/issues/58408#issuecomment-1423621323
|
|
|
|
// StdoutPipe/StderrPipe issues: https://pkg.go.dev/os/exec#Cmd.StdoutPipe
|
|
|
|
// Waiting for data: https://cs.opensource.google/go/go/+/refs/tags/go1.22.5:src/os/exec/exec.go;l=201
|
|
|
|
func runWithOutput(cmd *exec.Cmd, stdout, stderr io.Writer) error {
|
|
|
|
cmd.Stdout = struct{ io.Writer }{stdout}
|
|
|
|
cmd.Stderr = struct{ io.Writer }{stderr}
|
|
|
|
return cmd.Run()
|
|
|
|
}
|
|
|
|
|
|
|
|
// oomSniffer is a scanner that detects OOM errors in the output of a nodejs process.
|
|
|
|
type oomSniffer struct {
|
|
|
|
detected bool
|
|
|
|
timeout time.Duration
|
|
|
|
waitChan chan struct{}
|
|
|
|
}
|
|
|
|
|
|
|
|
func newOOMSniffer() *oomSniffer {
|
|
|
|
return &oomSniffer{
|
|
|
|
timeout: 15 * time.Second,
|
|
|
|
waitChan: make(chan struct{}),
|
2024-06-27 18:34:31 +00:00
|
|
|
}
|
2024-07-23 16:10:43 +00:00
|
|
|
}
|
2024-06-27 18:34:31 +00:00
|
|
|
|
2024-07-23 16:10:43 +00:00
|
|
|
func (o *oomSniffer) Detected() bool {
|
|
|
|
return o.detected
|
|
|
|
}
|
|
|
|
|
|
|
|
// Wait waits for the OOM sniffer to either:
|
|
|
|
// - detect an OOM error
|
|
|
|
// - the timeout to expire
|
|
|
|
// - the reader to be closed
|
|
|
|
//
|
|
|
|
// Call Wait to ensure we've read all the output from the scanned process after it exits.
|
|
|
|
func (o *oomSniffer) Wait() {
|
|
|
|
select {
|
|
|
|
case <-o.waitChan:
|
|
|
|
case <-time.After(o.timeout):
|
2024-06-27 18:34:31 +00:00
|
|
|
}
|
2024-07-23 16:10:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (o *oomSniffer) Message() string {
|
|
|
|
return "Detected a possible out of memory error. Consider increasing the memory available to the nodejs process " +
|
|
|
|
"by setting the `nodeargs` runtime option in Pulumi.yaml to `nodeargs: --max-old-space-size=<size>` where " +
|
|
|
|
"`<size>` is the maximum memory in megabytes that can be allocated to nodejs. " +
|
|
|
|
"See https://www.pulumi.com/docs/concepts/projects/project-file/#runtime-options"
|
|
|
|
}
|
|
|
|
|
|
|
|
func (o *oomSniffer) Scan(r io.Reader) {
|
|
|
|
scanner := bufio.NewScanner(r)
|
2024-06-27 18:34:31 +00:00
|
|
|
go func() {
|
2024-07-23 16:10:43 +00:00
|
|
|
for scanner.Scan() {
|
|
|
|
line := scanner.Text()
|
|
|
|
if !o.detected && strings.Contains(line, "<--- Last few GCs --->") /* "Normal" OOM output */ ||
|
|
|
|
// Because we hook into the debugger API, the OOM error message can be obscured by
|
|
|
|
// a failed assertion in the debugger https://github.com/pulumi/pulumi/issues/16596.
|
|
|
|
strings.Contains(line, "Check failed: needs_context && current_scope_ = closure_scope_") {
|
|
|
|
o.detected = true
|
|
|
|
close(o.waitChan)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
contract.IgnoreError(scanner.Err())
|
|
|
|
if !o.detected {
|
|
|
|
close(o.waitChan)
|
|
|
|
}
|
2024-06-27 18:34:31 +00:00
|
|
|
}()
|
|
|
|
}
|