2022-10-17 14:21:11 +00:00
|
|
|
// Copyright 2016-2023, Pulumi Corporation.
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2020-04-06 19:30:40 +00:00
|
|
|
"bytes"
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
"context"
|
|
|
|
"encoding/json"
|
2023-01-14 20:27:47 +00:00
|
|
|
"errors"
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
"flag"
|
|
|
|
"fmt"
|
2020-04-06 19:30:40 +00:00
|
|
|
"io"
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
"os"
|
|
|
|
"os/exec"
|
2023-08-29 15:42:31 +00:00
|
|
|
"os/signal"
|
2018-09-19 22:22:33 +00:00
|
|
|
"path/filepath"
|
2023-12-12 12:19:42 +00:00
|
|
|
"strconv"
|
2020-04-06 19:30:40 +00:00
|
|
|
"strings"
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
"syscall"
|
2022-06-06 12:28:00 +00:00
|
|
|
"time"
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
|
2020-04-06 19:30:40 +00:00
|
|
|
"github.com/blang/semver"
|
2021-06-11 02:57:18 +00:00
|
|
|
"github.com/opentracing/opentracing-go"
|
2023-04-26 20:11:15 +00:00
|
|
|
"golang.org/x/mod/modfile"
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
"google.golang.org/grpc"
|
2023-07-27 21:39:36 +00:00
|
|
|
"google.golang.org/grpc/codes"
|
|
|
|
"google.golang.org/grpc/status"
|
2024-01-17 09:35:20 +00:00
|
|
|
"google.golang.org/protobuf/types/known/emptypb"
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
|
2024-01-17 22:25:47 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/go/pulumi-language-go/v3/buildutil"
|
2024-01-17 14:56:18 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/go/pulumi-language-go/v3/goversion"
|
2022-09-06 23:44:29 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/constant"
|
2021-12-02 00:12:28 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
|
2023-06-28 16:02:04 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/slice"
|
2021-03-17 13:20:05 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
|
2021-03-17 13:20:05 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/executable"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/version"
|
2022-10-17 14:21:11 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
|
2021-03-17 13:20:05 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
|
|
|
pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
|
2022-10-17 14:21:11 +00:00
|
|
|
|
|
|
|
codegen "github.com/pulumi/pulumi/pkg/v3/codegen/go"
|
|
|
|
hclsyntax "github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax"
|
|
|
|
"github.com/pulumi/pulumi/pkg/v3/codegen/pcl"
|
|
|
|
"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
)
|
|
|
|
|
2022-09-15 16:10:09 +00:00
|
|
|
// This function takes a file target to specify where to compile to.
|
|
|
|
// If `outfile` is "", the binary is compiled to a new temporary file.
|
|
|
|
// This function returns the path of the file that was produced.
|
2022-10-04 08:58:01 +00:00
|
|
|
func compileProgram(programDirectory string, outfile string) (string, error) {
|
|
|
|
goFileSearchPattern := filepath.Join(programDirectory, "*.go")
|
2022-08-26 14:16:48 +00:00
|
|
|
if matches, err := filepath.Glob(goFileSearchPattern); err != nil || len(matches) == 0 {
|
2023-01-14 20:27:47 +00:00
|
|
|
return "", fmt.Errorf("Failed to find go files for 'go build' matching %s", goFileSearchPattern)
|
2022-08-26 14:16:48 +00:00
|
|
|
}
|
|
|
|
|
2022-09-14 21:48:09 +00:00
|
|
|
if outfile == "" {
|
2022-09-14 22:18:30 +00:00
|
|
|
// If no outfile is supplied, write the Go binary to a temporary file.
|
2022-09-15 16:10:09 +00:00
|
|
|
f, err := os.CreateTemp("", "pulumi-go.*")
|
2022-09-14 21:48:09 +00:00
|
|
|
if err != nil {
|
2023-01-14 20:27:47 +00:00
|
|
|
return "", fmt.Errorf("unable to create go program temp file: %w", err)
|
2022-09-14 21:48:09 +00:00
|
|
|
}
|
2020-03-18 19:41:45 +00:00
|
|
|
|
2022-09-14 21:48:09 +00:00
|
|
|
if err := f.Close(); err != nil {
|
2023-01-14 20:27:47 +00:00
|
|
|
return "", fmt.Errorf("unable to close go program temp file: %w", err)
|
2022-09-14 21:48:09 +00:00
|
|
|
}
|
|
|
|
outfile = f.Name()
|
2022-08-26 14:16:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
gobin, err := executable.FindExecutable("go")
|
2020-03-18 19:41:45 +00:00
|
|
|
if err != nil {
|
2023-01-14 20:27:47 +00:00
|
|
|
return "", fmt.Errorf("unable to find 'go' executable: %w", err)
|
2020-03-18 19:41:45 +00:00
|
|
|
}
|
2022-10-04 08:58:01 +00:00
|
|
|
logging.V(5).Infof("Attempting to build go program in %s with: %s build -o %s", programDirectory, gobin, outfile)
|
|
|
|
buildCmd := exec.Command(gobin, "build", "-o", outfile)
|
|
|
|
buildCmd.Dir = programDirectory
|
2022-08-26 14:16:48 +00:00
|
|
|
buildCmd.Stdout, buildCmd.Stderr = os.Stdout, os.Stderr
|
2020-03-18 19:41:45 +00:00
|
|
|
|
2022-08-26 14:16:48 +00:00
|
|
|
if err := buildCmd.Run(); err != nil {
|
2023-01-14 20:27:47 +00:00
|
|
|
return "", fmt.Errorf("unable to run `go build`: %w", err)
|
2020-03-18 19:41:45 +00:00
|
|
|
}
|
|
|
|
|
2022-08-26 14:16:48 +00:00
|
|
|
return outfile, nil
|
2020-03-18 19:41:45 +00:00
|
|
|
}
|
|
|
|
|
2023-04-18 22:47:13 +00:00
|
|
|
// runParams defines the command line arguments accepted by this program.
|
|
|
|
type runParams struct {
|
|
|
|
tracing string
|
|
|
|
engineAddress string
|
|
|
|
}
|
|
|
|
|
|
|
|
// parseRunParams parses the given arguments into a runParams structure,
|
|
|
|
// using the provided FlagSet.
|
|
|
|
func parseRunParams(flag *flag.FlagSet, args []string) (*runParams, error) {
|
|
|
|
var p runParams
|
|
|
|
flag.StringVar(&p.tracing, "tracing", "", "Emit tracing to a Zipkin-compatible tracing endpoint")
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
flag.String("binary", "", "[obsolete] Look on path for a binary executable with this name")
|
|
|
|
flag.String("buildTarget", "", "[obsolete] Path to use to output the compiled Pulumi Go program")
|
|
|
|
flag.String("root", "", "[obsolete] Project root path to use")
|
2023-04-18 22:47:13 +00:00
|
|
|
|
|
|
|
if err := flag.Parse(args); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Pluck out the engine so we can do logging, etc.
|
|
|
|
args = flag.Args()
|
|
|
|
if len(args) == 0 {
|
|
|
|
return nil, errors.New("missing required engine RPC address argument")
|
|
|
|
}
|
|
|
|
p.engineAddress = args[0]
|
|
|
|
|
|
|
|
return &p, nil
|
|
|
|
}
|
|
|
|
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
// Launches the language host, which in turn fires up an RPC server implementing the LanguageRuntimeServer endpoint.
|
|
|
|
func main() {
|
2023-04-18 22:47:13 +00:00
|
|
|
p, err := parseRunParams(flag.CommandLine, os.Args[1:])
|
|
|
|
if err != nil {
|
|
|
|
cmdutil.Exit(err)
|
|
|
|
}
|
|
|
|
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
logging.InitLogging(false, 0, false)
|
2023-04-18 22:47:13 +00:00
|
|
|
cmdutil.InitTracing("pulumi-language-go", "pulumi-language-go", p.tracing)
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
|
2023-04-18 22:47:13 +00:00
|
|
|
var cmd mainCmd
|
|
|
|
if err := cmd.Run(p); err != nil {
|
|
|
|
cmdutil.Exit(err)
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
}
|
2023-04-18 22:47:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type mainCmd struct {
|
2023-04-21 00:48:46 +00:00
|
|
|
Stdout io.Writer // == os.Stdout
|
|
|
|
Getwd func() (string, error) // == os.Getwd
|
2023-04-18 22:47:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (cmd *mainCmd) init() {
|
|
|
|
if cmd.Stdout == nil {
|
|
|
|
cmd.Stdout = os.Stdout
|
|
|
|
}
|
2023-04-21 00:48:46 +00:00
|
|
|
if cmd.Getwd == nil {
|
|
|
|
cmd.Getwd = os.Getwd
|
|
|
|
}
|
2023-04-18 22:47:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (cmd *mainCmd) Run(p *runParams) error {
|
|
|
|
cmd.init()
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
|
2023-04-21 00:48:46 +00:00
|
|
|
cwd, err := cmd.Getwd()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-08-29 15:42:31 +00:00
|
|
|
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
|
2022-06-06 12:28:00 +00:00
|
|
|
// map the context Done channel to the rpcutil boolean cancel channel
|
|
|
|
cancelChannel := make(chan bool)
|
|
|
|
go func() {
|
|
|
|
<-ctx.Done()
|
2023-08-29 15:42:31 +00:00
|
|
|
cancel() // deregister handler so we don't catch another interrupt
|
2022-06-06 12:28:00 +00:00
|
|
|
close(cancelChannel)
|
|
|
|
}()
|
2023-04-21 00:48:46 +00:00
|
|
|
err = rpcutil.Healthcheck(ctx, p.engineAddress, 5*time.Minute, cancel)
|
2022-06-06 12:28:00 +00:00
|
|
|
if err != nil {
|
2023-04-18 22:47:13 +00:00
|
|
|
return fmt.Errorf("could not start health check host RPC server: %w", err)
|
2022-09-14 21:48:09 +00:00
|
|
|
}
|
|
|
|
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
// Fire up a gRPC server, letting the kernel choose a free port.
|
2022-11-01 15:15:09 +00:00
|
|
|
handle, err := rpcutil.ServeWithOptions(rpcutil.ServeOptions{
|
|
|
|
Cancel: cancelChannel,
|
|
|
|
Init: func(srv *grpc.Server) error {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
host := newLanguageHost(p.engineAddress, cwd, p.tracing)
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
pulumirpc.RegisterLanguageRuntimeServer(srv, host)
|
|
|
|
return nil
|
|
|
|
},
|
2022-11-01 15:15:09 +00:00
|
|
|
Options: rpcutil.OpenTracingServerInterceptorOptions(nil),
|
|
|
|
})
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
if err != nil {
|
2023-04-18 22:47:13 +00:00
|
|
|
return fmt.Errorf("could not start language host RPC server: %w", err)
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Otherwise, print out the port so that the spawner knows how to reach us.
|
2023-04-18 22:47:13 +00:00
|
|
|
fmt.Fprintf(cmd.Stdout, "%d\n", handle.Port)
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
|
|
|
|
// And finally wait for the server to stop serving.
|
2022-11-01 15:15:09 +00:00
|
|
|
if err := <-handle.Done; err != nil {
|
2023-04-18 22:47:13 +00:00
|
|
|
return fmt.Errorf("language host RPC stopped serving: %w", err)
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
}
|
2023-04-18 22:47:13 +00:00
|
|
|
|
|
|
|
return nil
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// goLanguageHost implements the LanguageRuntimeServer interface for use as an API endpoint.
|
|
|
|
type goLanguageHost struct {
|
2022-12-14 19:20:26 +00:00
|
|
|
pulumirpc.UnsafeLanguageRuntimeServer // opt out of forward compat
|
|
|
|
|
2023-04-21 00:48:46 +00:00
|
|
|
cwd string
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
engineAddress string
|
|
|
|
tracing string
|
|
|
|
}
|
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
type goOptions struct {
|
|
|
|
// Look on path for a binary executable with this name.
|
|
|
|
binary string
|
|
|
|
// Path to use to output the compiled Pulumi Go program.
|
|
|
|
buildTarget string
|
|
|
|
}
|
|
|
|
|
|
|
|
func parseOptions(root string, options map[string]interface{}) (goOptions, error) {
|
|
|
|
var goOptions goOptions
|
|
|
|
if binary, ok := options["binary"]; ok {
|
|
|
|
if binary, ok := binary.(string); ok {
|
|
|
|
goOptions.binary = binary
|
|
|
|
} else {
|
|
|
|
return goOptions, errors.New("binary option must be a string")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if buildTarget, ok := options["buildTarget"]; ok {
|
|
|
|
if args, ok := buildTarget.(string); ok {
|
|
|
|
goOptions.buildTarget = args
|
|
|
|
} else {
|
|
|
|
return goOptions, errors.New("buildTarget option must be a string")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if goOptions.binary != "" && goOptions.buildTarget != "" {
|
|
|
|
return goOptions, errors.New("binary and buildTarget cannot both be specified")
|
|
|
|
}
|
|
|
|
|
|
|
|
return goOptions, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func newLanguageHost(engineAddress, cwd, tracing string) pulumirpc.LanguageRuntimeServer {
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
return &goLanguageHost{
|
|
|
|
engineAddress: engineAddress,
|
2023-04-21 00:48:46 +00:00
|
|
|
cwd: cwd,
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
tracing: tracing,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-07-11 22:00:40 +00:00
|
|
|
// modInfo is the useful portion of the output from
|
|
|
|
// 'go list -m -json' and 'go mod download -json'
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
// with respect to plugin acquisition.
|
2023-07-11 22:00:40 +00:00
|
|
|
// The listed fields are present in both command outputs.
|
|
|
|
//
|
|
|
|
// If we add fields that are only present in one or the other,
|
|
|
|
// we'll need to add a new struct type instead of re-using this.
|
2020-04-06 19:30:40 +00:00
|
|
|
type modInfo struct {
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
// Path is the module import path.
|
|
|
|
Path string
|
|
|
|
|
|
|
|
// Version of the module.
|
2020-04-06 19:30:40 +00:00
|
|
|
Version string
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
|
2023-07-11 22:00:40 +00:00
|
|
|
// Dir is the directory holding the source code of the module, if any.
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
Dir string
|
2020-04-06 19:30:40 +00:00
|
|
|
}
|
|
|
|
|
2023-07-11 22:00:40 +00:00
|
|
|
// findModuleSources finds the source code roots for the given modules.
|
|
|
|
//
|
|
|
|
// gobin is the path to the go binary to use.
|
|
|
|
// rootModuleDir is the path to the root directory of the program that may import the modules.
|
|
|
|
// It must contain the go.mod file for the program.
|
|
|
|
// modulePaths is a list of import paths for the modules to find.
|
|
|
|
//
|
|
|
|
// If $rootModuleDir/vendor exists, findModuleSources operates in vendor mode.
|
|
|
|
// In vendor mode, returned paths are inside the vendor directory exclusively.
|
|
|
|
func findModuleSources(ctx context.Context, gobin, rootModuleDir string, modulePaths []string) ([]modInfo, error) {
|
|
|
|
contract.Requiref(gobin != "", "gobin", "must not be empty")
|
|
|
|
contract.Requiref(rootModuleDir != "", "rootModuleDir", "must not be empty")
|
|
|
|
if len(modulePaths) == 0 {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// To find the source code for a module, we would typically use
|
|
|
|
// 'go list -m -json'.
|
|
|
|
// Its output includes, among other things:
|
|
|
|
//
|
|
|
|
// type Module struct {
|
|
|
|
// ...
|
|
|
|
// Dir string // directory holding local copy of files, if any
|
|
|
|
// ...
|
|
|
|
// }
|
|
|
|
//
|
|
|
|
// However, whether Dir is set or not depends on a few different factors.
|
|
|
|
//
|
|
|
|
// - If the module is not in the local module cache,
|
|
|
|
// then Dir is always empty.
|
|
|
|
// - If the module is imported by the current module, then Dir is set.
|
|
|
|
// - If the module is not imported,
|
|
|
|
// then Dir is set if we run with -mod=mod.
|
|
|
|
// - If the module is not imported and we run without -mod=mod,
|
|
|
|
// then:
|
|
|
|
// - If there's a vendor/ directory, then Dir is not set
|
|
|
|
// because we're running in vendor mode.
|
|
|
|
// - If there's no vendor/ directory, then Dir is set
|
|
|
|
// if we add the module to the module cache
|
|
|
|
// with `go mod download $path`.
|
|
|
|
//
|
|
|
|
// These are all corner cases that aren't fully specified,
|
|
|
|
// and may change between versions of Go.
|
|
|
|
//
|
|
|
|
// Therefore, the flow we use is:
|
|
|
|
//
|
|
|
|
// - Run 'go list -m -json $path1 $path2 ...'
|
|
|
|
// to make a first pass at getting module information.
|
|
|
|
// - If there's a vendor/ directory,
|
|
|
|
// use the module information from the vendor directory,
|
|
|
|
// skipping anything that's missing.
|
|
|
|
// We can't make requests to download modules in vendor mode.
|
|
|
|
// - Otherwise, for modules with missing Dir fields,
|
|
|
|
// run `go mod download -json $path` to download them to the module cache
|
|
|
|
// and get their locations.
|
|
|
|
|
|
|
|
modules, err := goListModules(ctx, gobin, rootModuleDir, modulePaths)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("go list: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// If there's a vendor directory, then we're in vendor mode.
|
|
|
|
// In vendor mode, Dir won't be set for any modules.
|
|
|
|
// Find these modules in the vendor directory.
|
|
|
|
vendorDir := filepath.Join(rootModuleDir, "vendor")
|
|
|
|
if _, err := os.Stat(vendorDir); err == nil {
|
|
|
|
newModules := modules[:0] // in-place filter
|
|
|
|
for _, module := range modules {
|
|
|
|
if module.Dir == "" {
|
|
|
|
vendoredModule := filepath.Join(vendorDir, module.Path)
|
|
|
|
if _, err := os.Stat(vendoredModule); err == nil {
|
|
|
|
module.Dir = vendoredModule
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// We can't download modules in vendor mode,
|
|
|
|
// so we'll skip any modules that aren't already in the vendor directory.
|
|
|
|
if module.Dir != "" {
|
|
|
|
newModules = append(newModules, module)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return newModules, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// We're not in vendor mode, so we can download modules and fill in missing directories.
|
|
|
|
var (
|
|
|
|
// Import paths of modules with no Dir field.
|
|
|
|
missingDirs []string
|
|
|
|
|
|
|
|
// Map from module path to index in modules.
|
|
|
|
moduleIndex = make(map[string]int, len(modules))
|
|
|
|
)
|
|
|
|
for i, module := range modules {
|
|
|
|
moduleIndex[module.Path] = i
|
|
|
|
if module.Dir == "" {
|
|
|
|
missingDirs = append(missingDirs, module.Path)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Fill in missing module directories with `go mod download`.
|
|
|
|
if len(missingDirs) > 0 {
|
|
|
|
missingMods, err := goModDownload(ctx, gobin, rootModuleDir, missingDirs)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("go mod download: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, m := range missingMods {
|
|
|
|
if m.Dir == "" {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// If this was a module we were missing,
|
|
|
|
// then we can fill in the directory now.
|
|
|
|
if idx, ok := moduleIndex[m.Path]; ok && modules[idx].Dir == "" {
|
|
|
|
modules[idx].Dir = m.Dir
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Any other modules with no Dir field can be discarded;
|
|
|
|
// we tried our best to find their source.
|
|
|
|
newModules := modules[:0] // in-place filter
|
|
|
|
for _, module := range modules {
|
|
|
|
if module.Dir != "" {
|
|
|
|
newModules = append(newModules, module)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return newModules, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Runs 'go list -m' on the given list of modules
|
|
|
|
// and reports information about them.
|
|
|
|
func goListModules(ctx context.Context, gobin, dir string, modulePaths []string) ([]modInfo, error) {
|
|
|
|
args := slice.Prealloc[string](len(modulePaths) + 3)
|
|
|
|
args = append(args, "list", "-m", "-json")
|
|
|
|
args = append(args, modulePaths...)
|
|
|
|
|
|
|
|
span, ctx := opentracing.StartSpanFromContext(ctx,
|
2024-04-19 06:20:33 +00:00
|
|
|
gobin+" list -m json",
|
2023-07-11 22:00:40 +00:00
|
|
|
opentracing.Tag{Key: "component", Value: "exec.Command"},
|
|
|
|
opentracing.Tag{Key: "command", Value: gobin},
|
|
|
|
opentracing.Tag{Key: "args", Value: args})
|
|
|
|
defer span.Finish()
|
|
|
|
|
|
|
|
cmd := exec.CommandContext(ctx, gobin, args...)
|
|
|
|
cmd.Dir = dir
|
|
|
|
cmd.Stderr = os.Stderr
|
|
|
|
|
|
|
|
stdout, err := cmd.StdoutPipe()
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("create stdout pipe: %w", err)
|
|
|
|
}
|
|
|
|
if err := cmd.Start(); err != nil {
|
|
|
|
return nil, fmt.Errorf("start command: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
var modules []modInfo
|
|
|
|
dec := json.NewDecoder(stdout)
|
|
|
|
for dec.More() {
|
|
|
|
var info modInfo
|
|
|
|
if err := dec.Decode(&info); err != nil {
|
|
|
|
return nil, fmt.Errorf("decode module info: %w", err)
|
|
|
|
}
|
|
|
|
modules = append(modules, info)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := cmd.Wait(); err != nil {
|
|
|
|
return nil, fmt.Errorf("wait for command: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return modules, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// goModDownload downloads the given modules to the module cache,
|
|
|
|
// reporting information about them in the returned modInfo.
|
|
|
|
func goModDownload(ctx context.Context, gobin, dir string, modulePaths []string) ([]modInfo, error) {
|
|
|
|
args := slice.Prealloc[string](len(modulePaths) + 3)
|
|
|
|
args = append(args, "mod", "download", "-json")
|
|
|
|
args = append(args, modulePaths...)
|
|
|
|
|
|
|
|
span, ctx := opentracing.StartSpanFromContext(ctx,
|
2024-04-19 06:20:33 +00:00
|
|
|
gobin+" mod download -json",
|
2023-07-11 22:00:40 +00:00
|
|
|
opentracing.Tag{Key: "component", Value: "exec.Command"},
|
|
|
|
opentracing.Tag{Key: "command", Value: gobin},
|
|
|
|
opentracing.Tag{Key: "args", Value: args})
|
|
|
|
defer span.Finish()
|
|
|
|
|
|
|
|
cmd := exec.CommandContext(ctx, gobin, args...)
|
|
|
|
cmd.Dir = dir
|
|
|
|
cmd.Stderr = os.Stderr
|
|
|
|
|
|
|
|
stdout, err := cmd.StdoutPipe()
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("create stdout pipe: %w", err)
|
|
|
|
}
|
|
|
|
if err := cmd.Start(); err != nil {
|
|
|
|
return nil, fmt.Errorf("start command: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
var modules []modInfo
|
|
|
|
dec := json.NewDecoder(stdout)
|
|
|
|
for dec.More() {
|
|
|
|
var info modInfo
|
|
|
|
if err := dec.Decode(&info); err != nil {
|
|
|
|
return nil, fmt.Errorf("decode module info: %w", err)
|
|
|
|
}
|
|
|
|
modules = append(modules, info)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := cmd.Wait(); err != nil {
|
|
|
|
return nil, fmt.Errorf("wait for command: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return modules, nil
|
|
|
|
}
|
|
|
|
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
// Returns the pulumi-plugin.json if found.
|
|
|
|
// If not found, then returns nil, nil.
|
|
|
|
//
|
|
|
|
// The lookup path for pulumi-plugin.json is:
|
|
|
|
//
|
|
|
|
// - m.Dir
|
|
|
|
// - m.Dir/go
|
|
|
|
// - m.Dir/go/*
|
|
|
|
//
|
|
|
|
// moduleRoot is the root directory of the module that imports this plugin.
|
|
|
|
// It is used to resolve the vendor directory.
|
|
|
|
// moduleRoot may be empty if unknown.
|
|
|
|
func (m *modInfo) readPulumiPluginJSON(moduleRoot string) (*plugin.PulumiPluginJSON, error) {
|
|
|
|
dir := m.Dir
|
2023-07-11 22:00:40 +00:00
|
|
|
contract.Assertf(dir != "", "module directory must be known")
|
2021-12-06 21:10:30 +00:00
|
|
|
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
paths := []string{
|
|
|
|
filepath.Join(dir, "pulumi-plugin.json"),
|
|
|
|
filepath.Join(dir, "go", "pulumi-plugin.json"),
|
|
|
|
}
|
|
|
|
if path, err := filepath.Glob(filepath.Join(dir, "go", "*", "pulumi-plugin.json")); err == nil {
|
|
|
|
paths = append(paths, path...)
|
2021-12-06 21:10:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, path := range paths {
|
|
|
|
plugin, err := plugin.LoadPulumiPluginJSON(path)
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
continue
|
|
|
|
}
|
2021-12-06 21:10:30 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
return plugin, nil
|
2021-12-02 00:12:28 +00:00
|
|
|
}
|
2021-12-06 21:10:30 +00:00
|
|
|
return nil, nil
|
2021-12-02 00:12:28 +00:00
|
|
|
}
|
2020-04-06 19:30:40 +00:00
|
|
|
|
2021-12-02 00:12:28 +00:00
|
|
|
func normalizeVersion(version string) (string, error) {
|
|
|
|
v, err := semver.ParseTolerant(version)
|
2020-04-06 19:30:40 +00:00
|
|
|
if err != nil {
|
2021-12-02 00:12:28 +00:00
|
|
|
return "", errors.New("module does not have semver compatible version")
|
2020-04-06 19:30:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// psuedoversions are commits that don't have a corresponding tag at the specified git hash
|
|
|
|
// https://golang.org/cmd/go/#hdr-Pseudo_versions
|
|
|
|
// pulumi-aws v1.29.1-0.20200403140640-efb5e2a48a86 (first commit after 1.29.0 release)
|
|
|
|
if buildutil.IsPseudoVersion(version) {
|
|
|
|
// no prior tag means there was never a release build
|
|
|
|
if v.Major == 0 && v.Minor == 0 && v.Patch == 0 {
|
2021-12-02 00:12:28 +00:00
|
|
|
return "", errors.New("invalid pseduoversion with no prior tag")
|
2020-04-06 19:30:40 +00:00
|
|
|
}
|
|
|
|
// patch is typically bumped from the previous tag when using pseudo version
|
|
|
|
// downgrade the patch by 1 to make sure we match a release that exists
|
|
|
|
patch := v.Patch
|
|
|
|
if patch > 0 {
|
|
|
|
patch--
|
|
|
|
}
|
|
|
|
version = fmt.Sprintf("v%v.%v.%v", v.Major, v.Minor, patch)
|
|
|
|
}
|
2021-12-02 00:12:28 +00:00
|
|
|
return version, nil
|
|
|
|
}
|
|
|
|
|
2023-05-17 17:40:18 +00:00
|
|
|
// getPlugin loads information about this plugin.
|
|
|
|
//
|
|
|
|
// moduleRoot is the root directory of the Go module that imports this plugin.
|
|
|
|
// It must hold the go.mod file and the vendor directory (if any).
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
func (m *modInfo) getPlugin(moduleRoot string) (*pulumirpc.PluginDependency, error) {
|
|
|
|
pulumiPlugin, err := m.readPulumiPluginJSON(moduleRoot)
|
2021-12-02 00:12:28 +00:00
|
|
|
if err != nil {
|
2022-10-09 14:58:33 +00:00
|
|
|
return nil, fmt.Errorf("failed to load pulumi-plugin.json: %w", err)
|
2021-12-02 00:12:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!strings.HasPrefix(m.Path, "github.com/pulumi/pulumi-") && pulumiPlugin == nil) ||
|
|
|
|
(pulumiPlugin != nil && !pulumiPlugin.Resource) {
|
|
|
|
return nil, errors.New("module is not a pulumi provider")
|
|
|
|
}
|
|
|
|
|
|
|
|
var name string
|
|
|
|
if pulumiPlugin != nil && pulumiPlugin.Name != "" {
|
|
|
|
name = pulumiPlugin.Name
|
|
|
|
} else {
|
|
|
|
// github.com/pulumi/pulumi-aws/sdk/... => aws
|
|
|
|
pluginPart := strings.Split(m.Path, "/")[2]
|
|
|
|
name = strings.SplitN(pluginPart, "-", 2)[1]
|
|
|
|
}
|
|
|
|
|
|
|
|
version := m.Version
|
|
|
|
if pulumiPlugin != nil && pulumiPlugin.Version != "" {
|
|
|
|
version = pulumiPlugin.Version
|
|
|
|
}
|
|
|
|
version, err = normalizeVersion(version)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
var server string
|
|
|
|
|
|
|
|
if pulumiPlugin != nil {
|
2021-12-15 18:41:44 +00:00
|
|
|
// There is no way to specify server without using `pulumi-plugin.json`.
|
2021-12-02 00:12:28 +00:00
|
|
|
server = pulumiPlugin.Server
|
|
|
|
}
|
2020-04-06 19:30:40 +00:00
|
|
|
|
|
|
|
plugin := &pulumirpc.PluginDependency{
|
|
|
|
Name: name,
|
|
|
|
Version: version,
|
|
|
|
Kind: "resource",
|
2021-12-02 00:12:28 +00:00
|
|
|
Server: server,
|
2020-04-06 19:30:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return plugin, nil
|
|
|
|
}
|
|
|
|
|
2023-05-17 17:40:18 +00:00
|
|
|
// Reads and parses the go.mod file for the program at the given path.
|
|
|
|
// Returns the parsed go.mod file and the path to the module directory.
|
|
|
|
// Relies on the 'go' command to find the go.mod file for this program.
|
|
|
|
func (host *goLanguageHost) loadGomod(gobin, programDir string) (modDir string, modFile *modfile.File, err error) {
|
|
|
|
// Get the path to the go.mod file.
|
|
|
|
// This may be different from the programDir if the Pulumi program
|
|
|
|
// is in a subdirectory of the Go module.
|
|
|
|
//
|
|
|
|
// The '-f {{.GoMod}}' specifies that the command should print
|
|
|
|
// just the path to the go.mod file.
|
|
|
|
//
|
|
|
|
// type Module struct {
|
|
|
|
// Path string // module path
|
|
|
|
// ...
|
|
|
|
// GoMod string // path to go.mod file
|
|
|
|
// }
|
|
|
|
//
|
|
|
|
// See 'go help list' for the full definition.
|
|
|
|
cmd := exec.Command(gobin, "list", "-m", "-f", "{{.GoMod}}")
|
2024-03-21 09:48:04 +00:00
|
|
|
// Disable GOWORK, we only want the one module found in this (or a parent) directory. Workspaces being
|
|
|
|
// enabled will cause "list -m" to list every module in the workspace.
|
|
|
|
cmd.Env = append(os.Environ(), "GOWORK=off")
|
2023-05-17 17:40:18 +00:00
|
|
|
cmd.Dir = programDir
|
|
|
|
cmd.Stderr = os.Stderr
|
|
|
|
out, err := cmd.Output()
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
if err != nil {
|
2023-05-17 17:40:18 +00:00
|
|
|
return "", nil, fmt.Errorf("go list -m: %w", err)
|
|
|
|
}
|
|
|
|
out = bytes.TrimSpace(out)
|
|
|
|
if len(out) == 0 {
|
|
|
|
// The 'go list' command above will exit successfully
|
|
|
|
// and return no output if the program is not in a Go module.
|
|
|
|
return "", nil, fmt.Errorf("no go.mod file found: %v", programDir)
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
}
|
|
|
|
|
2023-05-17 17:40:18 +00:00
|
|
|
modPath := string(out)
|
|
|
|
body, err := os.ReadFile(modPath)
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
if err != nil {
|
2023-05-17 17:40:18 +00:00
|
|
|
return "", nil, err
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
}
|
|
|
|
|
2023-05-17 17:40:18 +00:00
|
|
|
f, err := modfile.ParseLax(modPath, body, nil)
|
|
|
|
if err != nil {
|
|
|
|
return "", nil, fmt.Errorf("parse: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return filepath.Dir(modPath), f, nil
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
}
|
|
|
|
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
// GetRequiredPlugins computes the complete set of anticipated plugins required by a program.
|
2020-11-11 23:37:33 +00:00
|
|
|
// We're lenient here as this relies on the `go list` command and the use of modules.
|
|
|
|
// If the consumer insists on using some other form of dependency management tool like
|
|
|
|
// dep or glide, the list command fails with "go list -m: not using modules".
|
|
|
|
// However, we do enforce that go 1.14.0 or higher is installed.
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
func (host *goLanguageHost) GetRequiredPlugins(ctx context.Context,
|
2023-03-03 16:36:39 +00:00
|
|
|
req *pulumirpc.GetRequiredPluginsRequest,
|
|
|
|
) (*pulumirpc.GetRequiredPluginsResponse, error) {
|
2020-04-06 19:30:40 +00:00
|
|
|
logging.V(5).Infof("GetRequiredPlugins: Determining pulumi packages")
|
|
|
|
|
2020-04-09 23:16:10 +00:00
|
|
|
gobin, err := executable.FindExecutable("go")
|
2020-04-06 19:30:40 +00:00
|
|
|
if err != nil {
|
2023-01-14 20:27:47 +00:00
|
|
|
return nil, fmt.Errorf("couldn't find go binary: %w", err)
|
2020-04-06 19:30:40 +00:00
|
|
|
}
|
|
|
|
|
2020-11-11 23:37:33 +00:00
|
|
|
if err = goversion.CheckMinimumGoVersion(gobin); err != nil {
|
2020-11-11 23:01:38 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
moduleDir, gomod, err := host.loadGomod(gobin, req.Info.ProgramDirectory)
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
if err != nil {
|
|
|
|
// Don't fail if not using Go modules.
|
|
|
|
logging.V(5).Infof("GetRequiredPlugins: Error reading go.mod: %v", err)
|
|
|
|
return &pulumirpc.GetRequiredPluginsResponse{}, nil
|
|
|
|
}
|
|
|
|
|
2023-07-11 22:00:40 +00:00
|
|
|
modulePaths := slice.Prealloc[string](len(gomod.Require))
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
for _, req := range gomod.Require {
|
2023-07-11 22:00:40 +00:00
|
|
|
modulePaths = append(modulePaths, req.Mod.Path)
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
}
|
2021-06-11 02:57:18 +00:00
|
|
|
|
2023-07-11 22:00:40 +00:00
|
|
|
modInfos, err := findModuleSources(ctx, gobin, moduleDir, modulePaths)
|
2020-04-06 19:30:40 +00:00
|
|
|
if err != nil {
|
2023-07-11 22:00:40 +00:00
|
|
|
logging.V(5).Infof("GetRequiredPlugins: Error finding module sources: %v", err)
|
2020-11-11 23:37:33 +00:00
|
|
|
return &pulumirpc.GetRequiredPluginsResponse{}, nil
|
2020-04-06 19:30:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
plugins := []*pulumirpc.PluginDependency{}
|
2023-07-11 22:00:40 +00:00
|
|
|
for _, m := range modInfos {
|
2023-05-17 17:40:18 +00:00
|
|
|
plugin, err := m.getPlugin(moduleDir)
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
if err != nil {
|
2020-04-06 19:30:40 +00:00
|
|
|
logging.V(5).Infof(
|
|
|
|
"GetRequiredPlugins: Ignoring dependency: %s, version: %s, error: %s",
|
|
|
|
m.Path,
|
|
|
|
m.Version,
|
2023-12-20 15:54:06 +00:00
|
|
|
err,
|
2020-04-06 19:30:40 +00:00
|
|
|
)
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
continue
|
2020-04-06 19:30:40 +00:00
|
|
|
}
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
|
|
|
|
logging.V(5).Infof("GetRequiredPlugins: Found plugin name: %s, version: %s", plugin.Name, plugin.Version)
|
|
|
|
plugins = append(plugins, plugin)
|
2020-04-06 19:30:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return &pulumirpc.GetRequiredPluginsResponse{
|
|
|
|
Plugins: plugins,
|
|
|
|
}, nil
|
2018-09-19 22:22:33 +00:00
|
|
|
}
|
|
|
|
|
2022-09-06 23:44:29 +00:00
|
|
|
func runCmdStatus(cmd *exec.Cmd, env []string) (int, error) {
|
2022-08-26 14:16:48 +00:00
|
|
|
cmd.Env = env
|
|
|
|
cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
|
|
|
|
|
|
|
|
err := cmd.Run()
|
2023-01-14 20:27:47 +00:00
|
|
|
// The returned error is nil if the command runs, has no problems copying stdin, stdout, and stderr, and
|
|
|
|
// exits with a zero exit status.
|
|
|
|
if err == nil {
|
|
|
|
return 0, nil
|
|
|
|
}
|
2022-08-26 14:16:48 +00:00
|
|
|
|
|
|
|
// error handling
|
|
|
|
exiterr, ok := err.(*exec.ExitError)
|
|
|
|
if !ok {
|
2023-01-14 20:27:47 +00:00
|
|
|
return 0, fmt.Errorf("command errored unexpectedly: %w", err)
|
2022-08-26 14:16:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// retrieve the status code
|
|
|
|
status, ok := exiterr.Sys().(syscall.WaitStatus)
|
|
|
|
if !ok {
|
2023-01-14 20:27:47 +00:00
|
|
|
return 0, fmt.Errorf("program exited unexpectedly: %w", err)
|
2022-08-26 14:16:48 +00:00
|
|
|
}
|
|
|
|
|
2022-09-06 23:44:29 +00:00
|
|
|
return status.ExitStatus(), nil
|
|
|
|
}
|
|
|
|
|
2022-10-04 08:58:01 +00:00
|
|
|
func runProgram(pwd, bin string, env []string) *pulumirpc.RunResponse {
|
2022-09-06 23:44:29 +00:00
|
|
|
cmd := exec.Command(bin)
|
2022-10-04 08:58:01 +00:00
|
|
|
cmd.Dir = pwd
|
2022-09-06 23:44:29 +00:00
|
|
|
status, err := runCmdStatus(cmd, env)
|
|
|
|
if err != nil {
|
|
|
|
return &pulumirpc.RunResponse{
|
|
|
|
Error: err.Error(),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if status == 0 {
|
|
|
|
return &pulumirpc.RunResponse{}
|
|
|
|
}
|
|
|
|
|
|
|
|
// If the program ran, but returned an error,
|
|
|
|
// the error message should look as nice as possible.
|
|
|
|
if status == constant.ExitStatusLoggedError {
|
|
|
|
// program failed but sent an error to the engine
|
|
|
|
return &pulumirpc.RunResponse{
|
|
|
|
Bail: true,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// If the program ran, but exited with a non-zero and non-reserved error code.
|
|
|
|
// indicate to the user which exit code the program returned.
|
|
|
|
return &pulumirpc.RunResponse{
|
|
|
|
Error: fmt.Sprintf("program exited with non-zero exit code: %d", status),
|
|
|
|
}
|
2022-08-26 14:16:48 +00:00
|
|
|
}
|
|
|
|
|
2022-10-09 14:58:33 +00:00
|
|
|
// Run is RPC endpoint for LanguageRuntimeServer::Run
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
func (host *goLanguageHost) Run(ctx context.Context, req *pulumirpc.RunRequest) (*pulumirpc.RunResponse, error) {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
opts, err := parseOptions(req.Info.RootDirectory, req.Info.Options.AsMap())
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
// Create the environment we'll use to run the process. This is how we pass the RunInfo to the actual
|
|
|
|
// Go program runtime, to avoid needing any sort of program interface other than just a main entrypoint.
|
|
|
|
env, err := host.constructEnv(req)
|
|
|
|
if err != nil {
|
2023-01-14 20:27:47 +00:00
|
|
|
return nil, fmt.Errorf("failed to prepare environment: %w", err)
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
}
|
|
|
|
|
2022-08-26 14:16:48 +00:00
|
|
|
// the user can explicitly opt in to using a binary executable by specifying
|
|
|
|
// runtime.options.binary in the Pulumi.yaml
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
if opts.binary != "" {
|
|
|
|
bin, err := executable.FindExecutable(opts.binary)
|
2022-08-26 14:16:48 +00:00
|
|
|
if err != nil {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
return nil, fmt.Errorf("unable to find '%s' executable: %w", opts.binary, err)
|
2022-08-26 14:16:48 +00:00
|
|
|
}
|
2022-10-04 08:58:01 +00:00
|
|
|
return runProgram(req.Pwd, bin, env), nil
|
2018-09-19 22:22:33 +00:00
|
|
|
}
|
|
|
|
|
2022-09-06 23:44:29 +00:00
|
|
|
// feature flag to enable deprecated old behavior and use `go run`
|
2022-08-26 14:16:48 +00:00
|
|
|
if os.Getenv("PULUMI_GO_USE_RUN") != "" {
|
|
|
|
gobin, err := executable.FindExecutable("go")
|
|
|
|
if err != nil {
|
2023-01-14 20:27:47 +00:00
|
|
|
return nil, fmt.Errorf("unable to find 'go' executable: %w", err)
|
2022-08-26 14:16:48 +00:00
|
|
|
}
|
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
cmd := exec.Command(gobin, "run", req.Info.ProgramDirectory)
|
2023-04-21 00:48:46 +00:00
|
|
|
cmd.Dir = host.cwd
|
2022-09-06 23:44:29 +00:00
|
|
|
status, err := runCmdStatus(cmd, env)
|
|
|
|
if err != nil {
|
2022-08-26 14:16:48 +00:00
|
|
|
return &pulumirpc.RunResponse{
|
|
|
|
Error: err.Error(),
|
|
|
|
}, nil
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
}
|
|
|
|
|
2022-09-06 23:44:29 +00:00
|
|
|
// `go run` does not return the actual exit status of a program
|
|
|
|
// it only returns 2 non-zero exit statuses {1, 2}
|
|
|
|
// and it emits the exit status to stderr
|
|
|
|
if status != 0 {
|
|
|
|
return &pulumirpc.RunResponse{
|
|
|
|
Bail: true,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2022-08-26 14:16:48 +00:00
|
|
|
return &pulumirpc.RunResponse{}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// user did not specify a binary and we will compile and run the binary on-demand
|
|
|
|
logging.V(5).Infof("No prebuilt executable specified, attempting invocation via compilation")
|
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
program, err := compileProgram(req.Info.ProgramDirectory, opts.buildTarget)
|
2022-08-26 14:16:48 +00:00
|
|
|
if err != nil {
|
2023-01-14 20:27:47 +00:00
|
|
|
return nil, fmt.Errorf("error in compiling Go: %w", err)
|
2022-08-26 14:16:48 +00:00
|
|
|
}
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
if opts.buildTarget == "" {
|
2022-09-14 22:18:30 +00:00
|
|
|
// If there is no specified buildTarget, delete the temporary program after running it.
|
2022-09-14 21:48:09 +00:00
|
|
|
defer os.Remove(program)
|
|
|
|
}
|
2022-08-26 14:16:48 +00:00
|
|
|
|
2022-10-04 08:58:01 +00:00
|
|
|
return runProgram(req.Pwd, program, env), nil
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// constructEnv constructs an environment for a Go progam by enumerating all of the optional and non-optional
|
|
|
|
// arguments present in a RunRequest.
|
|
|
|
func (host *goLanguageHost) constructEnv(req *pulumirpc.RunRequest) ([]string, error) {
|
|
|
|
config, err := host.constructConfig(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2021-05-18 22:02:43 +00:00
|
|
|
configSecretKeys, err := host.constructConfigSecretKeys(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
|
2018-09-20 22:17:18 +00:00
|
|
|
env := os.Environ()
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
maybeAppendEnv := func(k, v string) {
|
|
|
|
if v != "" {
|
|
|
|
env = append(env, fmt.Sprintf("%s=%s", k, v))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-20 08:50:04 +00:00
|
|
|
maybeAppendEnv(pulumi.EnvOrganization, req.GetOrganization())
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
maybeAppendEnv(pulumi.EnvProject, req.GetProject())
|
|
|
|
maybeAppendEnv(pulumi.EnvStack, req.GetStack())
|
|
|
|
maybeAppendEnv(pulumi.EnvConfig, config)
|
2021-05-18 22:02:43 +00:00
|
|
|
maybeAppendEnv(pulumi.EnvConfigSecretKeys, configSecretKeys)
|
2023-12-12 12:19:42 +00:00
|
|
|
maybeAppendEnv(pulumi.EnvDryRun, strconv.FormatBool(req.GetDryRun()))
|
|
|
|
maybeAppendEnv(pulumi.EnvParallel, strconv.Itoa(int(req.GetParallel())))
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
maybeAppendEnv(pulumi.EnvMonitor, req.GetMonitorAddress())
|
|
|
|
maybeAppendEnv(pulumi.EnvEngine, host.engineAddress)
|
|
|
|
|
|
|
|
return env, nil
|
|
|
|
}
|
|
|
|
|
2019-12-16 22:51:02 +00:00
|
|
|
// constructConfig JSON-serializes the configuration data given as part of a RunRequest.
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
func (host *goLanguageHost) constructConfig(req *pulumirpc.RunRequest) (string, error) {
|
|
|
|
configMap := req.GetConfig()
|
|
|
|
if configMap == nil {
|
|
|
|
return "", nil
|
|
|
|
}
|
|
|
|
|
|
|
|
configJSON, err := json.Marshal(configMap)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
return string(configJSON), nil
|
|
|
|
}
|
|
|
|
|
2021-05-18 22:02:43 +00:00
|
|
|
// constructConfigSecretKeys JSON-serializes the list of keys that contain secret values given as part of
|
|
|
|
// a RunRequest.
|
|
|
|
func (host *goLanguageHost) constructConfigSecretKeys(req *pulumirpc.RunRequest) (string, error) {
|
|
|
|
configSecretKeys := req.GetConfigSecretKeys()
|
|
|
|
if configSecretKeys == nil {
|
|
|
|
return "[]", nil
|
|
|
|
}
|
|
|
|
|
|
|
|
configSecretKeysJSON, err := json.Marshal(configSecretKeys)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
return string(configSecretKeysJSON), nil
|
|
|
|
}
|
|
|
|
|
2024-01-17 09:35:20 +00:00
|
|
|
func (host *goLanguageHost) GetPluginInfo(ctx context.Context, req *emptypb.Empty) (*pulumirpc.PluginInfo, error) {
|
Support Pulumi programs written in Go
This adds rudimentary support for Pulumi programs written in Go. It
is not complete yet but the basic resource registration works.
Note that, stylistically speaking, Go is a bit different from our other
languages. This made it a bit easier to build this initial prototype,
since what we want is actually a rather thin veneer atop our existing
RPC interfaces. The lack of generics, however, adds some friction and
is something I'm continuing to hammer on; this will most likely lead to
little specialized types (e.g. StringOutput) once the dust settles.
There are two primary components:
1) A new language host, `pulumi-language-go`, which is responsible for
communicating with the engine through the usual gRPC interfaces.
Because Go programs are pre-compiled, it very simply loads a binary
with the same name as the project.
2) A client SDK library that Pulumi programs bind against. This exports
the core resource types -- including assets -- properties -- including
output properties -- and configuration.
Most remaining TODOs are marked as such in the code, and this will not
be merged until they have been addressed, and some better tests written.
2018-06-03 17:37:26 +00:00
|
|
|
return &pulumirpc.PluginInfo{
|
|
|
|
Version: version.Version,
|
|
|
|
}, nil
|
|
|
|
}
|
2022-04-03 14:54:59 +00:00
|
|
|
|
|
|
|
func (host *goLanguageHost) InstallDependencies(
|
2023-03-03 16:36:39 +00:00
|
|
|
req *pulumirpc.InstallDependenciesRequest, server pulumirpc.LanguageRuntime_InstallDependenciesServer,
|
|
|
|
) error {
|
2022-10-04 08:58:01 +00:00
|
|
|
closer, stdout, stderr, err := rpcutil.MakeInstallDependenciesStreams(server, req.IsTerminal)
|
2022-04-03 14:54:59 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
// best effort close, but we try an explicit close and error check at the end as well
|
|
|
|
defer closer.Close()
|
|
|
|
|
|
|
|
stdout.Write([]byte("Installing dependencies...\n\n"))
|
|
|
|
|
|
|
|
gobin, err := executable.FindExecutable("go")
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if err = goversion.CheckMinimumGoVersion(gobin); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-05-04 18:59:37 +00:00
|
|
|
cmd := exec.Command(gobin, "mod", "tidy", "-compat=1.18")
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
cmd.Dir = req.Info.ProgramDirectory
|
2022-04-03 14:54:59 +00:00
|
|
|
cmd.Env = os.Environ()
|
|
|
|
cmd.Stdout, cmd.Stderr = stdout, stderr
|
|
|
|
|
|
|
|
if err := cmd.Run(); err != nil {
|
|
|
|
return fmt.Errorf("`go mod tidy` failed to install dependencies: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
stdout.Write([]byte("Finished installing dependencies\n\n"))
|
|
|
|
|
all: Fix revive issues
Fixes the following issues found by revive
included in the latest release of golangci-lint.
Full list of issues:
**pkg**
```
backend/display/object_diff.go:47:10: superfluous-else: if block ends with a break statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
backend/display/object_diff.go:716:12: redefines-builtin-id: redefinition of the built-in function delete (revive)
backend/display/object_diff.go:742:14: redefines-builtin-id: redefinition of the built-in function delete (revive)
backend/display/object_diff.go:983:10: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (revive)
backend/httpstate/backend.go:1814:4: redefines-builtin-id: redefinition of the built-in function cap (revive)
backend/httpstate/backend.go:1824:5: redefines-builtin-id: redefinition of the built-in function cap (revive)
backend/httpstate/client/client.go:444:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
backend/httpstate/client/client.go:455:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
cmd/pulumi/org.go:113:4: if-return: redundant if ...; err != nil check, just return error instead. (revive)
cmd/pulumi/util.go:216:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
codegen/docs/gen.go:428:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
codegen/hcl2/model/expression.go:2151:5: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:151:2: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:329:3: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:381:5: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/nodejs/gen.go:1367:5: redefines-builtin-id: redefinition of the built-in function copy (revive)
codegen/python/gen_program_expressions.go:136:2: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/python/gen_program_expressions.go:142:3: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/report/report.go:126:6: redefines-builtin-id: redefinition of the built-in function panic (revive)
codegen/schema/docs_test.go:210:10: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
codegen/schema/schema.go:790:2: redefines-builtin-id: redefinition of the built-in type any (revive)
codegen/schema/schema.go:793:4: redefines-builtin-id: redefinition of the built-in type any (revive)
resource/deploy/plan.go:506:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
resource/deploy/snapshot_test.go:59:3: redefines-builtin-id: redefinition of the built-in function copy (revive)
resource/deploy/state_builder.go:108:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
```
**sdk**
```
go/common/resource/plugin/context.go:142:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
go/common/resource/plugin/plugin.go:142:12: superfluous-else: if block ends with a break statement, so drop this else and outdent its block (revive)
go/common/resource/properties_diff.go:114:2: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:117:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:122:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:127:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:132:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/util/deepcopy/copy.go:30:1: redefines-builtin-id: redefinition of the built-in function copy (revive)
go/common/workspace/creds.go:242:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi-language-go/main.go:569:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi-language-go/main.go:706:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi/run_test.go:925:2: redefines-builtin-id: redefinition of the built-in type any (revive)
go/pulumi/run_test.go:933:3: redefines-builtin-id: redefinition of the built-in type any (revive)
nodejs/cmd/pulumi-language-nodejs/main.go:778:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/cmd/pulumi-language-python/main.go:1011:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/cmd/pulumi-language-python/main.go:863:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/python.go:230:2: redefines-builtin-id: redefinition of the built-in function print (revive)
```
**tests**
```
integration/integration_util_test.go:282:11: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
```
2023-03-20 23:48:02 +00:00
|
|
|
return closer.Close()
|
2022-04-03 14:54:59 +00:00
|
|
|
}
|
2022-07-25 11:35:16 +00:00
|
|
|
|
2024-01-17 09:35:20 +00:00
|
|
|
func (host *goLanguageHost) About(ctx context.Context, req *emptypb.Empty) (*pulumirpc.AboutResponse, error) {
|
2022-08-15 13:55:04 +00:00
|
|
|
getResponse := func(execString string, args ...string) (string, string, error) {
|
|
|
|
ex, err := executable.FindExecutable(execString)
|
|
|
|
if err != nil {
|
|
|
|
return "", "", fmt.Errorf("could not find executable '%s': %w", execString, err)
|
|
|
|
}
|
|
|
|
cmd := exec.Command(ex, args...)
|
2023-04-21 00:48:46 +00:00
|
|
|
cmd.Dir = host.cwd
|
2022-08-15 13:55:04 +00:00
|
|
|
var out []byte
|
|
|
|
if out, err = cmd.Output(); err != nil {
|
|
|
|
cmd := ex
|
|
|
|
if len(args) != 0 {
|
|
|
|
cmd += " " + strings.Join(args, " ")
|
|
|
|
}
|
|
|
|
return "", "", fmt.Errorf("failed to execute '%s'", cmd)
|
|
|
|
}
|
|
|
|
return ex, strings.TrimSpace(string(out)), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
goexe, version, err := getResponse("go", "version")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return &pulumirpc.AboutResponse{
|
|
|
|
Executable: goexe,
|
|
|
|
Version: version,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2022-07-25 11:35:16 +00:00
|
|
|
func (host *goLanguageHost) GetProgramDependencies(
|
2023-03-03 16:36:39 +00:00
|
|
|
ctx context.Context, req *pulumirpc.GetProgramDependenciesRequest,
|
|
|
|
) (*pulumirpc.GetProgramDependenciesResponse, error) {
|
2023-05-17 17:40:18 +00:00
|
|
|
gobin, err := executable.FindExecutable("go")
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("couldn't find go binary: %w", err)
|
|
|
|
}
|
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
_, gomod, err := host.loadGomod(gobin, req.Info.ProgramDirectory)
|
2022-08-15 13:55:04 +00:00
|
|
|
if err != nil {
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
return nil, fmt.Errorf("load go.mod: %w", err)
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
|
2023-06-28 16:02:04 +00:00
|
|
|
result := slice.Prealloc[*pulumirpc.DependencyInfo](len(gomod.Require))
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
for _, d := range gomod.Require {
|
|
|
|
if !d.Indirect || req.TransitiveDependencies {
|
2022-08-15 13:55:04 +00:00
|
|
|
datum := pulumirpc.DependencyInfo{
|
feat(go/host): Support vendored dependencies
The Go language host cannot resolve dependencies or plugins if a Pulumi
program vendors its dependencies.
BACKGROUND
The GetRequiredPlugins and GetProgramDependencies methods of the Go
language host rely on the following two commands:
go list -m -mod=mod all
go list -m -mod=mod ...
# '...' means current module and its descendants
GetRequiredPlugins additionally searches the source directories for each
returned module for pulumi-plugin.json files at a pre-determined paths.
$module/pulumi-plugin.json
$module/go/pulumi-plugin.json
$module/go/*/pulumi-plugin.json
This works for most Pulumi programs, except those that vendor private
dependencies with 'go mod vendor'.
For those programs, the above commands fail because -mod=mod forces them
to run in module mode, and their private dependencies are not accessible
in module mode (because they are not exposed publicly).
We use the -mod=mod flag to force 'go list' to run in module mode
because otherwise, it will automatically use vendor mode if a vendor
directory is present. However, in vendor mode, the two 'go list'
commands above are not supported.
The following links add more context on why, but in short:
vendor does not have enough information for the general 'go list'.
- https://stackoverflow.com/a/60660593,
- https://github.com/golang/go/issues/35589#issuecomment-554488544
In short,
- list all with -mod=mod fails because the dependency is private
- list without -mod=mod will use vendor mode
- vendor mode doesn't support the listing all
SOLUTION
Drop the -mod=mod flag so that 'go list' can decide whether to run in
module mode or vendor mode.
However, instead of running it with 'all' or '...',
pass in a list of dependencies extracted from the go.mod.
go list -m import/path1 import/path2 # ...
This operation is completely offline in vendor mode
so it can list information about private dependencies too.
This alone isn't enough though because in vendor mode,
the JSON output does not include the module root directory.
E.g.
% go list -mod=vendor -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"GoVersion": "1.18"
}
# Versus
% go list -mod=mod -json -m github.com/pulumi/pulumi/sdk/v3
{
"Path": "github.com/pulumi/pulumi/sdk/v3",
"Version": "v3.55.0",
"Time": "2023-02-14T11:04:22Z",
"Dir": "[...]/go/pkg/mod/github.com/pulumi/pulumi/sdk/v3@v3.55.0",
"GoMod": "[...]/go/pkg/mod/cache/download/github.com/pulumi/pulumi/sdk/v3/@v/v3.55.0.mod",
"GoVersion": "1.18"
}
Therefore, we have to manually calculate the path for each module root.
That's easy enough: vendor/$importPath.
Lastly, since GetProgramDependencies only needs a dependency list,
it now extracts information from the go.mod without calling 'go list'.
TESTING
Adds a variant of the test added in #12715 that verifies the
functionality with vendoring. It removes the sources for the
dependencies to simulate private dependencies. The new test fails
without the accompanying change.
The fix was further manually verified against the reproduction included
in #12526.
% cd go-output
% pulumi plugin rm -a -y
% pulumi preview
Previewing update (abhinav):
Downloading plugin: 15.19 MiB / 15.19 MiB [=========================] 100.00% 0s
[resource plugin random-4.8.2] installing
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
% pulumi plugin ls
NAME KIND VERSION SIZE INSTALLED LAST USED
random resource 4.8.2 33 MB 26 seconds ago 26 seconds ago
TOTAL plugin cache size: 33 MB
Note that the version of random (4.8.2) is what's specified in the
go.mod, not the latest release (v4.12.1).
% grep pulumi-random go.mod
github.com/pulumi/pulumi-random/sdk/v4 v4.8.2
With the plugin downloaded, I ran this again without an internet
connection.
% pulumi preview
Previewing update (abhinav):
Type Name Plan
+ pulumi:pulumi:Stack go-output-abhinav create
+ └─ random:index:RandomId rrr create
Resources:
+ 2 to create
This means that if the dependencies are vendored, and the plugin is
already available, we won't make additional network requests, which also
addresses #7089.
Resolves #12526
Resolves #7089
2023-04-20 23:27:39 +00:00
|
|
|
Name: d.Mod.Path,
|
|
|
|
Version: d.Mod.Version,
|
2022-08-15 13:55:04 +00:00
|
|
|
}
|
|
|
|
result = append(result, &datum)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return &pulumirpc.GetProgramDependenciesResponse{
|
|
|
|
Dependencies: result,
|
|
|
|
}, nil
|
2022-07-25 11:35:16 +00:00
|
|
|
}
|
2022-10-04 08:58:01 +00:00
|
|
|
|
|
|
|
func (host *goLanguageHost) RunPlugin(
|
2023-03-03 16:36:39 +00:00
|
|
|
req *pulumirpc.RunPluginRequest, server pulumirpc.LanguageRuntime_RunPluginServer,
|
|
|
|
) error {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
logging.V(5).Infof("Attempting to run go plugin in %s", req.Info.ProgramDirectory)
|
2022-10-04 08:58:01 +00:00
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
program, err := compileProgram(req.Info.ProgramDirectory, "")
|
2022-10-04 08:58:01 +00:00
|
|
|
if err != nil {
|
2023-01-14 20:27:47 +00:00
|
|
|
return fmt.Errorf("error in compiling Go: %w", err)
|
2022-10-04 08:58:01 +00:00
|
|
|
}
|
|
|
|
defer os.Remove(program)
|
|
|
|
|
|
|
|
closer, stdout, stderr, err := rpcutil.MakeRunPluginStreams(server, false)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
// best effort close, but we try an explicit close and error check at the end as well
|
|
|
|
defer closer.Close()
|
|
|
|
|
|
|
|
cmd := exec.Command(program, req.Args...)
|
|
|
|
cmd.Dir = req.Pwd
|
|
|
|
cmd.Env = req.Env
|
|
|
|
cmd.Stdout, cmd.Stderr = stdout, stderr
|
|
|
|
|
|
|
|
if err = cmd.Run(); err != nil {
|
|
|
|
if exiterr, ok := err.(*exec.ExitError); ok {
|
|
|
|
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
|
|
|
|
err = server.Send(&pulumirpc.RunPluginResponse{
|
|
|
|
Output: &pulumirpc.RunPluginResponse_Exitcode{Exitcode: int32(status.ExitStatus())},
|
|
|
|
})
|
|
|
|
} else {
|
2023-01-14 20:27:47 +00:00
|
|
|
err = fmt.Errorf("program exited unexpectedly: %w", exiterr)
|
2022-10-04 08:58:01 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
return fmt.Errorf("problem executing plugin program (could not run language executor): %w", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
all: Fix revive issues
Fixes the following issues found by revive
included in the latest release of golangci-lint.
Full list of issues:
**pkg**
```
backend/display/object_diff.go:47:10: superfluous-else: if block ends with a break statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
backend/display/object_diff.go:716:12: redefines-builtin-id: redefinition of the built-in function delete (revive)
backend/display/object_diff.go:742:14: redefines-builtin-id: redefinition of the built-in function delete (revive)
backend/display/object_diff.go:983:10: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (revive)
backend/httpstate/backend.go:1814:4: redefines-builtin-id: redefinition of the built-in function cap (revive)
backend/httpstate/backend.go:1824:5: redefines-builtin-id: redefinition of the built-in function cap (revive)
backend/httpstate/client/client.go:444:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
backend/httpstate/client/client.go:455:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
cmd/pulumi/org.go:113:4: if-return: redundant if ...; err != nil check, just return error instead. (revive)
cmd/pulumi/util.go:216:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
codegen/docs/gen.go:428:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
codegen/hcl2/model/expression.go:2151:5: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:151:2: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:329:3: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:381:5: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/nodejs/gen.go:1367:5: redefines-builtin-id: redefinition of the built-in function copy (revive)
codegen/python/gen_program_expressions.go:136:2: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/python/gen_program_expressions.go:142:3: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/report/report.go:126:6: redefines-builtin-id: redefinition of the built-in function panic (revive)
codegen/schema/docs_test.go:210:10: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
codegen/schema/schema.go:790:2: redefines-builtin-id: redefinition of the built-in type any (revive)
codegen/schema/schema.go:793:4: redefines-builtin-id: redefinition of the built-in type any (revive)
resource/deploy/plan.go:506:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
resource/deploy/snapshot_test.go:59:3: redefines-builtin-id: redefinition of the built-in function copy (revive)
resource/deploy/state_builder.go:108:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
```
**sdk**
```
go/common/resource/plugin/context.go:142:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
go/common/resource/plugin/plugin.go:142:12: superfluous-else: if block ends with a break statement, so drop this else and outdent its block (revive)
go/common/resource/properties_diff.go:114:2: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:117:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:122:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:127:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:132:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/util/deepcopy/copy.go:30:1: redefines-builtin-id: redefinition of the built-in function copy (revive)
go/common/workspace/creds.go:242:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi-language-go/main.go:569:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi-language-go/main.go:706:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi/run_test.go:925:2: redefines-builtin-id: redefinition of the built-in type any (revive)
go/pulumi/run_test.go:933:3: redefines-builtin-id: redefinition of the built-in type any (revive)
nodejs/cmd/pulumi-language-nodejs/main.go:778:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/cmd/pulumi-language-python/main.go:1011:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/cmd/pulumi-language-python/main.go:863:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/python.go:230:2: redefines-builtin-id: redefinition of the built-in function print (revive)
```
**tests**
```
integration/integration_util_test.go:282:11: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
```
2023-03-20 23:48:02 +00:00
|
|
|
return closer.Close()
|
2022-10-04 08:58:01 +00:00
|
|
|
}
|
2022-10-17 14:21:11 +00:00
|
|
|
|
|
|
|
func (host *goLanguageHost) GenerateProject(
|
2023-05-19 21:41:27 +00:00
|
|
|
ctx context.Context, req *pulumirpc.GenerateProjectRequest,
|
|
|
|
) (*pulumirpc.GenerateProjectResponse, error) {
|
2023-07-27 09:27:07 +00:00
|
|
|
loader, err := schema.NewLoaderClient(req.LoaderTarget)
|
2022-10-17 14:21:11 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2023-08-31 16:35:21 +00:00
|
|
|
var extraOptions []pcl.BindOption
|
2023-06-23 14:19:02 +00:00
|
|
|
if !req.Strict {
|
2023-07-13 13:16:06 +00:00
|
|
|
extraOptions = append(extraOptions, pcl.NonStrictBindOptions()...)
|
2023-06-23 00:42:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
program, diags, err := pcl.BindDirectory(req.SourceDirectory, loader, extraOptions...)
|
2022-10-17 14:21:11 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-05-26 10:32:19 +00:00
|
|
|
|
2023-08-31 16:35:21 +00:00
|
|
|
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
|
2023-06-07 17:17:59 +00:00
|
|
|
if diags.HasErrors() {
|
2023-05-26 10:32:19 +00:00
|
|
|
return &pulumirpc.GenerateProjectResponse{
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
}, nil
|
2022-10-17 14:21:11 +00:00
|
|
|
}
|
2023-06-07 17:17:59 +00:00
|
|
|
if program == nil {
|
2024-04-19 06:20:33 +00:00
|
|
|
return nil, errors.New("internal error: program was nil")
|
2023-06-07 17:17:59 +00:00
|
|
|
}
|
2022-10-17 14:21:11 +00:00
|
|
|
|
|
|
|
var project workspace.Project
|
|
|
|
if err := json.Unmarshal([]byte(req.Project), &project); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2023-08-03 10:40:05 +00:00
|
|
|
err = codegen.GenerateProject(req.TargetDirectory, project, program, req.LocalDependencies)
|
2022-10-17 14:21:11 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2023-05-26 10:32:19 +00:00
|
|
|
return &pulumirpc.GenerateProjectResponse{
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
}, nil
|
2022-10-17 14:21:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (host *goLanguageHost) GenerateProgram(
|
2023-05-19 21:41:27 +00:00
|
|
|
ctx context.Context, req *pulumirpc.GenerateProgramRequest,
|
|
|
|
) (*pulumirpc.GenerateProgramResponse, error) {
|
2023-07-27 09:27:07 +00:00
|
|
|
loader, err := schema.NewLoaderClient(req.LoaderTarget)
|
2022-10-17 14:21:11 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
parser := hclsyntax.NewParser()
|
|
|
|
// Load all .pp files in the directory
|
|
|
|
for path, contents := range req.Source {
|
|
|
|
err = parser.ParseFile(strings.NewReader(contents), path)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
diags := parser.Diagnostics
|
|
|
|
if diags.HasErrors() {
|
|
|
|
return nil, diags
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-08-31 16:35:21 +00:00
|
|
|
program, diags, err := pcl.BindProgram(parser.Files, pcl.Loader(loader))
|
2022-10-17 14:21:11 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-06-07 17:17:59 +00:00
|
|
|
|
2023-08-31 16:35:21 +00:00
|
|
|
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
|
|
|
|
if diags.HasErrors() {
|
2023-06-07 17:17:59 +00:00
|
|
|
return &pulumirpc.GenerateProgramResponse{
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
if program == nil {
|
2024-04-19 06:20:33 +00:00
|
|
|
return nil, errors.New("internal error: program was nil")
|
2022-10-17 14:21:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
files, diags, err := codegen.GenerateProgram(program)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-08-31 16:35:21 +00:00
|
|
|
rpcDiagnostics = append(rpcDiagnostics, plugin.HclDiagnosticsToRPCDiagnostics(diags)...)
|
2022-10-17 14:21:11 +00:00
|
|
|
|
|
|
|
return &pulumirpc.GenerateProgramResponse{
|
|
|
|
Source: files,
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (host *goLanguageHost) GeneratePackage(
|
2023-05-19 21:41:27 +00:00
|
|
|
ctx context.Context, req *pulumirpc.GeneratePackageRequest,
|
|
|
|
) (*pulumirpc.GeneratePackageResponse, error) {
|
2022-10-17 14:21:11 +00:00
|
|
|
if len(req.ExtraFiles) > 0 {
|
|
|
|
return nil, errors.New("overlays are not supported for Go")
|
|
|
|
}
|
|
|
|
|
2023-07-27 09:27:07 +00:00
|
|
|
loader, err := schema.NewLoaderClient(req.LoaderTarget)
|
2022-10-17 14:21:11 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
var spec schema.PackageSpec
|
|
|
|
err = json.Unmarshal([]byte(req.Schema), &spec)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
pkg, diags, err := schema.BindSpec(spec, loader)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-12-05 17:47:52 +00:00
|
|
|
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
|
2022-10-17 14:21:11 +00:00
|
|
|
if diags.HasErrors() {
|
2023-12-05 17:47:52 +00:00
|
|
|
return &pulumirpc.GeneratePackageResponse{
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
}, nil
|
2022-10-17 14:21:11 +00:00
|
|
|
}
|
|
|
|
files, err := codegen.GeneratePackage("pulumi-language-go", pkg)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
for filename, data := range files {
|
2023-08-31 16:35:21 +00:00
|
|
|
outPath := filepath.Join(req.Directory, filename)
|
2022-10-17 14:21:11 +00:00
|
|
|
|
|
|
|
err := os.MkdirAll(filepath.Dir(outPath), 0o700)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("could not create output directory %s: %w", filepath.Dir(filename), err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = os.WriteFile(outPath, data, 0o600)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("could not write output file %s: %w", filename, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-05 17:47:52 +00:00
|
|
|
return &pulumirpc.GeneratePackageResponse{
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
}, nil
|
2022-10-17 14:21:11 +00:00
|
|
|
}
|
2023-07-27 21:39:36 +00:00
|
|
|
|
|
|
|
func (host *goLanguageHost) Pack(ctx context.Context, req *pulumirpc.PackRequest) (*pulumirpc.PackResponse, error) {
|
|
|
|
return nil, status.Errorf(codes.Unimplemented, "method Pack not implemented")
|
|
|
|
}
|