pulumi/sdk/nodejs/cmd/pulumi-language-nodejs/main_test.go

610 lines
18 KiB
Go
Raw Permalink Normal View History

2018-05-22 19:43:36 +00:00
// Copyright 2016-2018, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
2018-05-22 19:43:36 +00:00
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"syscall"
"testing"
"time"
"github.com/blang/semver"
"github.com/pulumi/pulumi/sdk/v3/nodejs/npm"
pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestArgumentConstruction(t *testing.T) {
t.Parallel()
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
info := &pulumirpc.ProgramInfo{
RootDirectory: "/foo/bar",
ProgramDirectory: "/foo/bar",
EntryPoint: ".",
}
t.Run("DryRun-NoArguments", func(t *testing.T) {
t.Parallel()
host := &nodeLanguageHost{}
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
rr := &pulumirpc.RunRequest{DryRun: true, Info: info}
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
args := host.constructArguments(rr, "", "", "")
assert.Contains(t, args, "--dry-run")
assert.NotContains(t, args, "true")
})
t.Run("OptionalArgs-PassedIfSpecified", func(t *testing.T) {
t.Parallel()
host := &nodeLanguageHost{}
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
rr := &pulumirpc.RunRequest{Project: "foo", Info: info}
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
args := strings.Join(host.constructArguments(rr, "", "", ""), " ")
assert.Contains(t, args, "--project foo")
})
t.Run("OptionalArgs-NotPassedIfNotSpecified", func(t *testing.T) {
t.Parallel()
host := &nodeLanguageHost{}
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
rr := &pulumirpc.RunRequest{Info: info}
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
args := strings.Join(host.constructArguments(rr, "", "", ""), " ")
assert.NotContains(t, args, "--stack")
})
t.Run("DotIfProgramNotSpecified", func(t *testing.T) {
t.Parallel()
host := &nodeLanguageHost{}
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
rr := &pulumirpc.RunRequest{Info: info}
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
args := strings.Join(host.constructArguments(rr, "", "", ""), " ")
assert.Contains(t, args, ".")
})
t.Run("ProgramIfProgramSpecified", func(t *testing.T) {
t.Parallel()
host := &nodeLanguageHost{}
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
rr := &pulumirpc.RunRequest{
Program: "foobar",
Info: &pulumirpc.ProgramInfo{
RootDirectory: "/foo/bar",
ProgramDirectory: "/foo/bar",
EntryPoint: "foobar",
},
}
Move InstallDependencies to the language plugin (#9294) * Move InstallDependencies to the language plugin This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself. This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this. We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli. An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well. The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC. The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given? * Just warn if pty fails to open * Add tests and return real tty files * Add to CHANGELOG * lint * format * Test strings * Log pty opening for trace debugging * s/Hack/Workaround * Use termios * Tweak terminal test * lint * Fix windows build
2022-04-03 14:54:59 +00:00
args := strings.Join(host.constructArguments(rr, "", "", ""), " ")
assert.Contains(t, args, "foobar")
})
}
func TestConfig(t *testing.T) {
t.Parallel()
t.Run("Config-Empty", func(t *testing.T) {
t.Parallel()
host := &nodeLanguageHost{}
rr := &pulumirpc.RunRequest{Project: "foo"}
str, err := host.constructConfig(rr)
assert.NoError(t, err)
assert.JSONEq(t, "{}", str)
})
}
func TestCompatibleVersions(t *testing.T) {
t.Parallel()
cases := []struct {
a string
b string
compatible bool
errmsg string
}{
{"0.17.1", "0.16.2", false, "Differing major or minor versions are not supported."},
{"0.17.1", "1.0.0", true, ""},
{"1.0.0", "0.17.1", true, ""},
{"1.13.0", "1.13.0", true, ""},
{"1.1.1", "1.13.0", true, ""},
{"1.13.0", "1.1.1", true, ""},
{"1.1.0", "2.1.0", true, ""},
{"2.1.0", "1.1.0", true, ""},
{"1.1.0", "2.0.0-beta1", true, ""},
{"2.0.0-beta1", "1.1.0", true, ""},
{"2.1.0", "3.1.0", false, "Differing major versions are not supported."},
{"0.16.1", "1.0.0", false, "Differing major or minor versions are not supported."},
}
for _, c := range cases {
compatible, errmsg := compatibleVersions(semver.MustParse(c.a), semver.MustParse(c.b))
assert.Equal(t, c.errmsg, errmsg)
assert.Equal(t, c.compatible, compatible)
}
}
func TestGetRequiredPlugins(t *testing.T) {
t.Parallel()
dir := t.TempDir()
files := []struct {
path string
content string
}{
{
filepath.Join(dir, "node_modules", "@pulumi", "foo", "package.json"),
`{ "name": "@pulumi/foo", "version": "1.2.3", "pulumi": { "resource": true } }`,
},
{
filepath.Join(dir, "node_modules", "@pulumi", "bar", "package.json"),
`{ "name": "@pulumi/bar", "version": "4.5.6", "pulumi": { "resource": true } }`,
},
{
filepath.Join(dir, "node_modules", "@pulumi", "baz", "package.json"),
`{ "name": "@pulumi/baz", "version": "4.5.6", "pulumi": { "resource": false } }`,
},
{
filepath.Join(dir, "node_modules", "malformed", "tests", "malformed_test", "package.json"),
`{`,
},
}
for _, file := range files {
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
err := os.MkdirAll(filepath.Dir(file.path), 0o755)
2023-01-23 19:53:29 +00:00
require.NoError(t, err)
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
err = os.WriteFile(file.path, []byte(file.content), 0o600)
2023-01-23 19:53:29 +00:00
require.NoError(t, err)
}
host := &nodeLanguageHost{}
resp, err := host.GetRequiredPlugins(context.Background(), &pulumirpc.GetRequiredPluginsRequest{
Program: dir,
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
Info: &pulumirpc.ProgramInfo{
RootDirectory: dir,
ProgramDirectory: dir,
EntryPoint: ".",
},
})
2023-01-23 19:53:29 +00:00
require.NoError(t, err)
actual := make(map[string]string)
for _, plugin := range resp.GetPlugins() {
actual[plugin.Name] = plugin.Version
}
assert.Equal(t, map[string]string{
"foo": "v1.2.3",
"bar": "v4.5.6",
}, actual)
}
func TestGetRequiredPluginsSymlinkCycles(t *testing.T) {
t.Parallel()
dir := t.TempDir()
files := []struct {
path string
content string
}{
{
filepath.Join(dir, "node_modules", "@pulumi", "foo", "package.json"),
`{ "name": "@pulumi/foo", "version": "1.2.3", "pulumi": { "resource": true } }`,
},
{
filepath.Join(dir, "node_modules", "@pulumi", "bar", "package.json"),
`{ "name": "@pulumi/bar", "version": "4.5.6", "pulumi": { "resource": true } }`,
},
{
filepath.Join(dir, "node_modules", "@pulumi", "baz", "package.json"),
`{ "name": "@pulumi/baz", "version": "4.5.6", "pulumi": { "resource": false } }`,
},
{
filepath.Join(dir, "node_modules", "malformed", "tests", "malformed_test", "package.json"),
`{`,
},
}
for _, file := range files {
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
err := os.MkdirAll(filepath.Dir(file.path), 0o755)
require.NoError(t, err)
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
err = os.WriteFile(file.path, []byte(file.content), 0o600)
require.NoError(t, err)
}
// Add a symlink cycle in
err := os.Symlink(filepath.Join(dir, "node_modules"), filepath.Join(dir, "node_modules", "@node_modules"))
require.NoError(t, err)
host := &nodeLanguageHost{}
resp, err := host.GetRequiredPlugins(context.Background(), &pulumirpc.GetRequiredPluginsRequest{
Program: dir,
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
Info: &pulumirpc.ProgramInfo{
RootDirectory: dir,
ProgramDirectory: dir,
EntryPoint: ".",
},
})
require.NoError(t, err)
actual := make(map[string]string)
for _, plugin := range resp.GetPlugins() {
actual[plugin.Name] = plugin.Version
}
assert.Equal(t, map[string]string{
"foo": "v1.2.3",
"bar": "v4.5.6",
}, actual)
}
2023-01-23 22:02:26 +00:00
func TestGetRequiredPluginsSymlinkCycles2(t *testing.T) {
t.Parallel()
dir := filepath.Join(t.TempDir(), "testdir")
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
err := os.Mkdir(dir, 0o755)
2023-01-23 22:02:26 +00:00
require.NoError(t, err)
files := []struct {
path string
content string
}{
{
filepath.Join(dir, "node_modules", "@pulumi", "foo", "package.json"),
`{ "name": "@pulumi/foo", "version": "1.2.3", "pulumi": { "resource": true } }`,
},
{
filepath.Join(dir, "node_modules", "@pulumi", "bar", "package.json"),
`{ "name": "@pulumi/bar", "version": "4.5.6", "pulumi": { "resource": true } }`,
},
{
filepath.Join(dir, "node_modules", "@pulumi", "baz", "package.json"),
`{ "name": "@pulumi/baz", "version": "4.5.6", "pulumi": { "resource": false } }`,
},
{
filepath.Join(dir, "node_modules", "malformed", "tests", "malformed_test", "package.json"),
`{`,
},
}
for _, file := range files {
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
err := os.MkdirAll(filepath.Dir(file.path), 0o755)
2023-01-23 22:02:26 +00:00
require.NoError(t, err)
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
err = os.WriteFile(file.path, []byte(file.content), 0o600)
2023-01-23 22:02:26 +00:00
require.NoError(t, err)
}
// Add a symlink cycle in
err = os.Symlink(filepath.Join("..", ".."), filepath.Join(dir, "node_modules", "@node_modules"))
require.NoError(t, err)
host := &nodeLanguageHost{}
resp, err := host.GetRequiredPlugins(context.Background(), &pulumirpc.GetRequiredPluginsRequest{
2023-01-23 22:02:26 +00:00
Program: dir,
Pass root and main info to language host methods (#14654) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is two changes rolled together in a way. Firstly passing some of the data that we pass on language runtime startup to also pass it to Run/GetRequiredPlugins/etc. This is needed for matrix testing, as we only get to start the language runtime up once for that but want to execute multiple programs with it. I feel it's also a little more consistent as we use the language runtimes in other contexts (codegen) where there isn't really a root directory, and aren't any options (and if we did do options the options for codegen are not going to be the same as for execution). It also means we can reuse a language host for shimless and substack programs, as before they heavily relied on their current working directory to calculate paths, and obviosly could only take one set of options at startup. Imagine a shimless python package + a python root program, that would have needed two startups of the python language host to deal with, this unblocks it so we can make the engine smarter and only use one. Secondly renaming some of the fields we pass to Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very descriptive and had pretty non-obvious documentation: ``` string pwd = 3; // the program's working directory. string program = 4; // the path to the program to execute. ``` `pwd` will remain, although probably rename it to `working_directory` at some point, because while today we always start programs up with the working directory equal to the program directory that definitely is going to change in the future (at least for MLCs and substack programs). But the name `pwd` doesn't make it clear that this was intended to be the working directory _and_ the directory which contains the program. `program` was in fact nearly always ".", and if it wasn't that it was just a filename. The engine never sent a path for `program` (although we did have some unit tests to check how that worked for the nodejs and python hosts). These are now replaced by a new structure with (I think) more clearly named and documented fields (see ProgramInfo in langauge.proto). The engine still sends the old data for now, we need to update dotnet/yaml/java before we break the old interface and give Virtus Labs a chance to update [besom](https://github.com/VirtusLab/besom). ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-12-10 17:30:51 +00:00
Info: &pulumirpc.ProgramInfo{
RootDirectory: dir,
ProgramDirectory: dir,
EntryPoint: ".",
},
2023-01-23 22:02:26 +00:00
})
require.NoError(t, err)
actual := make(map[string]string)
for _, plugin := range resp.GetPlugins() {
actual[plugin.Name] = plugin.Version
}
assert.Equal(t, map[string]string{
"foo": "v1.2.3",
"bar": "v4.5.6",
}, actual)
}
func TestGetRequiredPluginsNestedPolicyPack(t *testing.T) {
t.Parallel()
dir := filepath.Join(t.TempDir(), "testdir")
err := os.Mkdir(dir, 0o755)
require.NoError(t, err)
files := []struct {
path string
content string
}{
{
filepath.Join(dir, "node_modules", "@pulumi", "foo", "package.json"),
`{ "name": "@pulumi/foo", "version": "1.2.3", "pulumi": { "resource": true } }`,
},
{
filepath.Join(dir, "node_modules", "@pulumi", "bar", "package.json"),
`{ "name": "@pulumi/bar", "version": "4.5.6", "pulumi": { "resource": true } }`,
},
{
filepath.Join(dir, "policy", "PulumiPolicy.yaml"),
`name: my-policy`,
},
{
filepath.Join(dir, "policy", "node_modules", "@pulumi", "baz", "package.json"),
`{ "name": "@pulumi/baz", "version": "7.8.9", "pulumi": { "resource": true } }`,
},
}
for _, file := range files {
err := os.MkdirAll(filepath.Dir(file.path), 0o755)
require.NoError(t, err)
err = os.WriteFile(file.path, []byte(file.content), 0o600)
require.NoError(t, err)
}
host := &nodeLanguageHost{}
resp, err := host.GetRequiredPlugins(context.Background(), &pulumirpc.GetRequiredPluginsRequest{
Program: dir,
Info: &pulumirpc.ProgramInfo{
RootDirectory: dir,
ProgramDirectory: dir,
EntryPoint: ".",
},
})
require.NoError(t, err)
actual := make(map[string]string)
for _, plugin := range resp.GetPlugins() {
actual[plugin.Name] = plugin.Version
}
assert.Equal(t, map[string]string{
"foo": "v1.2.3",
"bar": "v4.5.6",
// baz: v7.8.9 is not included because it is in a nested policy pack
}, actual)
}
func TestParseOptions(t *testing.T) {
t.Parallel()
opts, err := parseOptions(nil)
require.NoError(t, err)
require.Equal(t, npm.AutoPackageManager, opts.packagemanager)
_, err = parseOptions(map[string]interface{}{
"typescript": 123,
})
require.ErrorContains(t, err, "typescript option must be a boolean")
_, err = parseOptions(map[string]interface{}{
"packagemanager": "poetry",
})
require.ErrorContains(t, err, "packagemanager option must be one of")
for _, tt := range []struct {
input string
expected npm.PackageManagerType
}{
{"auto", npm.AutoPackageManager},
{"npm", npm.NpmPackageManager},
{"yarn", npm.YarnPackageManager},
{"pnpm", npm.PnpmPackageManager},
} {
opts, err = parseOptions(map[string]interface{}{
"packagemanager": tt.input,
})
require.NoError(t, err)
require.Equal(t, tt.expected, opts.packagemanager)
}
}
// Nodejs sometimes sets stdout/stderr to non-blocking mode. When a nodejs subprocess is directly
// handed the go process's stdout/stderr file descriptors, nodejs's non-blocking configuration goes
// unnoticed by go, and a write from go can result in an error `write /dev/stdout: resource
// temporarily unavailable`. See runWithOutput for more details.
func TestNonblockingStdout(t *testing.T) {
// Regression test for https://github.com/pulumi/pulumi/issues/16503
t.Parallel()
script := `import os, time
os.set_blocking(1, False) # set stdout to non-blocking
time.sleep(3)
`
// Create a named pipe to use as stdout
tmp := os.TempDir()
p := filepath.Join(tmp, "fake-stdout")
err := syscall.Mkfifo(p, 0o644)
defer os.Remove(p)
require.NoError(t, err)
// Open fd without O_NONBLOCK, ensuring that os.NewFile does not return a pollable file.
// When our python script changes the file to non-blocking, Go does not notice and continues to
// expect the file to be blocking, and we can trigger the bug.
fd, err := syscall.Open(p, syscall.O_CREAT|syscall.O_RDWR, 0o644)
require.NoError(t, err)
fakeStdout := os.NewFile(uintptr(fd), p)
defer fakeStdout.Close()
require.NotNil(t, fakeStdout)
cmd := exec.Command("python3", "-c", script)
var done bool
go func() {
time.Sleep(2 * time.Second)
for !done {
s := "....................\n"
n, err := fakeStdout.Write([]byte(s))
require.NoError(t, err)
require.Equal(t, n, len(s))
}
}()
require.NoError(t, runWithOutput(cmd, fakeStdout, os.Stderr))
done = true
}
type slowWriter struct {
nWrites *int
}
func (s slowWriter) Write(b []byte) (int, error) {
time.Sleep(100 * time.Millisecond)
l := len(b)
*s.nWrites += l
return l, nil
}
func TestRunWithOutputDoesNotMissData(t *testing.T) {
// This test ensures that runWithOutput writes all the data from the command and does not miss
// any data that might be buffered when the command exits.
t.Parallel()
// Write `o` to stdout 100 times at 10 ms interval, followed by `x\n`
// Write `e` to stderr 100 times at 10 ms interval, followed by `x\n`
script := `let i = 0;
let interval = setInterval(() => {
process.stdout.write("o");
process.stderr.write("e");
i++;
if (i == 100) {
process.stdout.write("x\n");
process.stderr.write("x\n");
clearInterval(interval);
}
}, 10)
`
cmd := exec.Command("node", "-e", script)
stdout := slowWriter{nWrites: new(int)}
stderr := slowWriter{nWrites: new(int)}
require.NoError(t, runWithOutput(cmd, stdout, stderr))
require.Equal(t, 100+2 /* "x\n" */, *stdout.nWrites)
require.Equal(t, 100+2 /* "x\n" */, *stderr.nWrites)
}
//nolint:paralleltest // mutates environment variables
func TestUseFnm(t *testing.T) {
// Set $PATH to to $TMPDIR/bin so that no `fnm` executable can be found.
tmpDir := t.TempDir()
require.NoError(t, os.MkdirAll(filepath.Join(tmpDir, "bin"), 0o755))
t.Setenv("PATH", filepath.Join(tmpDir, "bin"))
_, err := useFnm(tmpDir)
require.ErrorIs(t, err, errFnmNotFound)
// Add a fake fnm binary to $TMPDIR/bin for the rest of the tests.
//nolint:gosec // we want this file to be executable
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "bin", "fnm"), []byte("#!/bin/sh\nexit 0;\n"), 0o700))
t.Run("no version files", func(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
_, err := useFnm(tmpDir)
require.ErrorIs(t, err, errVersionFileNotFound)
})
t.Run(".node-version in cwd", func(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, ".node-version"), []byte("22.7.3"), 0o600))
version, err := useFnm(tmpDir)
require.NoError(t, err)
require.Equal(t, "22.7.3", version)
})
t.Run(".nvmrc in cwd", func(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, ".nvmrc"), []byte("20.1.1"), 0o600))
version, err := useFnm(tmpDir)
require.NoError(t, err)
require.Equal(t, "20.1.1", version)
})
t.Run(".nvmrc & .node-version in cwd", func(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
// .nvmrc should take precedence
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, ".nvmrc"), []byte("20.1.1"), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, ".node-version"), []byte("22.7.3"), 0o600))
version, err := useFnm(tmpDir)
require.NoError(t, err)
require.Equal(t, "20.1.1", version)
})
t.Run(".node-version in parent folder", func(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
tmpDirNested := filepath.Join(tmpDir, "nested")
require.NoError(t, os.MkdirAll(tmpDirNested, 0o700))
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, ".node-version"), []byte("20.1.1"), 0o600))
version, err := useFnm(tmpDirNested)
require.NoError(t, err)
require.Equal(t, "20.1.1", version)
})
t.Run(".node-version in cwd & parent", func(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
tmpDirNested := filepath.Join(tmpDir, "nested")
require.NoError(t, os.MkdirAll(tmpDirNested, 0o700))
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, ".node-version"), []byte("20.1.1"), 0o600))
// This should take precedence over the parent folder's .node-version
require.NoError(t, os.WriteFile(filepath.Join(tmpDirNested, ".node-version"), []byte("20.7.3"), 0o600))
version, err := useFnm(tmpDirNested)
require.NoError(t, err)
require.Equal(t, "20.7.3", version)
})
}
//nolint:paralleltest // mutates environment variables
func TestNodeInstall(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
tmpDir := t.TempDir()
t.Setenv("PATH", filepath.Join(tmpDir, "bin"))
fmt.Println(os.Getenv("PATH"))
// There's no fnm executable in PATH, installNodeVersion is a no-op
stdout := &bytes.Buffer{}
err := installNodeVersion(tmpDir, stdout)
require.ErrorIs(t, err, errFnmNotFound)
// Add a mock fnm executable to $tmp/bin. For each execution, the mock fnm executable will
// append a line with its arguments to a file. We read back the file to verify that it was
// called with the expected arguments.
require.NoError(t, os.MkdirAll(filepath.Join(tmpDir, "bin"), 0o755))
outPath := filepath.Join(tmpDir, "out.txt")
script := fmt.Sprintf("#!/bin/sh\necho $@ >> %s\n", outPath)
//nolint:gosec // we want this file to be executable
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "bin", "fnm"), []byte(script), 0o700))
// There's no .node-version or .nvmrc file, so the binary should not be called.
// We expect the file written by our mock fnm executable to not exist.
stdout = &bytes.Buffer{}
err = installNodeVersion(tmpDir, stdout)
require.Error(t, err, errVersionFileNotFound)
// Create a .node-version file
// The mock fnm executable should be called with a command to install the requested version,
// and a command to set the default version to this version.
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, ".node-version"), []byte("20.1.2"), 0o600))
stdout = &bytes.Buffer{}
err = installNodeVersion(tmpDir, stdout)
require.NoError(t, err)
b, err := os.ReadFile(outPath)
require.NoError(t, err)
commands := strings.Split(strings.TrimSpace(string(b)), "\n")
require.Equal(t, "install 20.1.2 --progress never", commands[0])
require.Equal(t, "alias 20.1.2 default", commands[1])
}