pulumi/pkg/codegen/schema/loader.go

368 lines
9.4 KiB
Go
Raw Normal View History

// Copyright 2016-2022, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package schema
import (
"bytes"
"errors"
"fmt"
"io"
"os"
"sync"
"time"
"github.com/edsrzf/mmap-go"
"github.com/natefinch/atomic"
"github.com/blang/semver"
"github.com/segmentio/encoding/json"
pkgWorkspace "github.com/pulumi/pulumi/pkg/v3/workspace"
"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
Add an envar to disable automatic provider installation (#14083) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is primarily for the providers team to enable during builds so they can have more confidence about reproducibility of builds (especially examples conversion), but I imagine some customers would enable this as well. Fixes #14086 ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-10-03 15:35:23 +00:00
"github.com/pulumi/pulumi/sdk/v3/go/common/env"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
"github.com/pulumi/pulumi/sdk/v3/go/common/tokens"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
)
type Loader interface {
LoadPackage(pkg string, version *semver.Version) (*Package, error)
}
type ReferenceLoader interface {
Loader
LoadPackageReference(pkg string, version *semver.Version) (PackageReference, error)
}
type pluginLoader struct {
m sync.RWMutex
host plugin.Host
entries map[string]PackageReference
cacheOptions pluginLoaderCacheOptions
}
// Caching options intended for benchmarking or debugging:
type pluginLoaderCacheOptions struct {
// useEntriesCache enables in-memory re-use of packages
disableEntryCache bool
// useFileCache enables skipping plugin loading when possible and caching JSON schemas to files
disableFileCache bool
// useMmap enables the use of memory mapped IO to avoid copying the JSON schema
disableMmap bool
}
func NewPluginLoader(host plugin.Host) ReferenceLoader {
return &pluginLoader{
host: host,
entries: map[string]PackageReference{},
}
}
func newPluginLoaderWithOptions(host plugin.Host, cacheOptions pluginLoaderCacheOptions) ReferenceLoader {
return &pluginLoader{
host: host,
entries: map[string]PackageReference{},
cacheOptions: cacheOptions,
}
}
func (l *pluginLoader) getPackage(key string) (PackageReference, bool) {
if l.cacheOptions.disableEntryCache {
return nil, false
}
p, ok := l.entries[key]
return p, ok
}
func (l *pluginLoader) setPackage(key string, p PackageReference) PackageReference {
if l.cacheOptions.disableEntryCache {
return p
}
if p, ok := l.entries[key]; ok {
return p
}
l.entries[key] = p
return p
}
func (l *pluginLoader) LoadPackage(pkg string, version *semver.Version) (*Package, error) {
ref, err := l.LoadPackageReference(pkg, version)
if err != nil {
return nil, err
}
return ref.Definition()
}
var ErrGetSchemaNotImplemented = getSchemaNotImplemented{}
type getSchemaNotImplemented struct{}
func (f getSchemaNotImplemented) Error() string {
return "it looks like GetSchema is not implemented"
}
func schemaIsEmpty(schemaBytes []byte) bool {
// A non-empty schema is any that contains non-whitespace, non brace characters.
//
// Some providers implemented GetSchema initially by returning text matching the regular
// expression: "\s*\{\s*\}\s*". This handles those cases while not strictly checking that braces
// match or reading the whole document.
for _, v := range schemaBytes {
if v != ' ' && v != '\t' && v != '\r' && v != '\n' && v != '{' && v != '}' {
return false
}
}
return true
}
func (l *pluginLoader) LoadPackageReference(pkg string, version *semver.Version) (PackageReference, error) {
2022-07-19 02:47:31 +00:00
if pkg == "pulumi" {
return DefaultPulumiPackage.Reference(), nil
2022-07-19 02:47:31 +00:00
}
l.m.Lock()
defer l.m.Unlock()
key := packageIdentity(pkg, version)
if p, ok := l.getPackage(key); ok {
return p, nil
}
schemaBytes, version, err := l.loadSchemaBytes(pkg, version)
if err != nil {
return nil, err
}
if schemaIsEmpty(schemaBytes) {
return nil, getSchemaNotImplemented{}
}
var spec PartialPackageSpec
if _, err := json.Parse(schemaBytes, &spec, json.ZeroCopy); err != nil {
return nil, err
}
// Insert a version into the spec if the package does not provide one or if the
// existing version is less than the provided one
if version != nil {
setVersion := true
if spec.PackageInfoSpec.Version != "" {
vSemver, err := semver.Make(spec.PackageInfoSpec.Version)
if err == nil {
if vSemver.Compare(*version) == 1 {
setVersion = false
}
}
}
if setVersion {
spec.PackageInfoSpec.Version = version.String()
}
}
Add matrix testing (#13705) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Adds the first pass of matrix testing. Matrix testing allows us to define tests once in pulumi/pulumi via PCL and then run those tests against each language plugin to verify code generation and runtime correctness. Rather than packing matrix tests and all the associated data and machinery into the CLI itself we define a new Go package at cmd/pulumi-test-lanaguage. This depends on pkg and runs the deployment engine in a unique way for matrix tests but it is running the proper deployment engine with a proper backend (always filestate, using $TEMP). Currently only NodeJS is hooked up to run these tests, and all the code for that currently lives in sdk/nodejs/cmd/pulumi-language-nodejs/language_test.go. I expect we'll move that helper code to sdk/go/common and use it in each language plugin to run the tests in the same way. This first pass includes 3 simple tests: * l1-empty that runs an empty PCL file and checks just a stack is created * l1-output-bool that runs a PCL program that returns two stack outputs of `true` and `false * l2-resource-simple that runs a PCL program creating a simple resource with a single bool property These tests are themselves tested with a mock language runtime. This verifies the behavior of the matrix test framework for both correct and incorrect language hosts (that is some the mock language runtimes purposefully cause errors or compute the wrong result). There are a number of things missing from from the core framework still, but I feel don't block getting this first pass merged and starting to be used. 1. The tests can not currently run in parallel. That is calling RunLanguageTest in parallel will break things. This is due to two separate problems. Firstly is that the SDK snapshot's are not safe to write in parallel (when PULUMI_ACCEPT is true), this should be fairly easy to fix by doing a write to dst-{random} and them atomic move to dst. Secondly is that the deployment engine itself has mutable global state, short term we should probably just lock around that part RunLanguageTest, long term it would be good to clean that up. 2. We need a way to verify "preview" behavior, I think this is probably just a variation of the tests that would call `stack.Preview` and not pass a snapshot to `assert`. 3. stdout, stderr and log messages are returned in bulk at the end of the test. Plus there are a couple of calls to the language runtime that don't correctly thread stdout/stderr to use and so default to the process `os.Stdout/Stderr`. stdout/stderr streaming shows up in a load of other places as well so I'm thinking of a clean way to handle all of them together. Log message streaming we can probably do by just turning RunLanguageTest to a streaming grpc call. ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. --> --------- Co-authored-by: Abhinav Gupta <abhinav@pulumi.com>
2023-09-13 15:17:46 +00:00
p, err := ImportPartialSpec(spec, nil, l)
if err != nil {
return nil, err
}
return l.setPackage(key, p), nil
}
func LoadPackageReference(loader Loader, pkg string, version *semver.Version) (PackageReference, error) {
var ref PackageReference
var err error
if refLoader, ok := loader.(ReferenceLoader); ok {
ref, err = refLoader.LoadPackageReference(pkg, version)
} else {
p, pErr := loader.LoadPackage(pkg, version)
err = pErr
if err == nil {
ref = p.Reference()
}
}
if err != nil {
return nil, err
}
if pkg != ref.Name() || version != nil && ref.Version() != nil && !ref.Version().Equals(*version) {
if l, ok := loader.(*pluginLoader); ok {
return nil, fmt.Errorf("req: %s@%v: entries: %v (returned %s@%v)", pkg, version,
l.entries, ref.Name(), ref.Version())
}
return nil, fmt.Errorf("loader returned %s@%v: expected %s@%v", ref.Name(), ref.Version(), pkg, version)
}
return ref, nil
}
func (l *pluginLoader) loadSchemaBytes(pkg string, version *semver.Version) ([]byte, *semver.Version, error) {
Schema loader made to respect PULUMI_DEBUG_PROVIDERS (#15526) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> With this change pulumi-yaml can pick up local provider and attach to it from PULUMI_DEBUG_PROVIDERS for the purposes of schema resolution, which enables using non-existent test-only providers. Before the change it would fail hard trying to download it. ## Checklist - [ ] I have run `make tidy` to update any new dependencies - [ ] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. --> Co-authored-by: Thomas Gummerer <t.gummerer@gmail.com>
2024-03-04 21:54:05 +00:00
attachPort, err := plugin.GetProviderAttachPort(tokens.Package(pkg))
if err != nil {
return nil, nil, err
}
// If PULUMI_DEBUG_PROVIDERS requested an attach port, skip caching and workspace
// interaction and load the schema directly from the given port.
if attachPort != nil {
schemaBytes, provider, err := l.loadPluginSchemaBytes(pkg, version)
if err != nil {
return nil, nil, fmt.Errorf("Error loading schema from plugin: %w", err)
}
if version == nil {
info, err := provider.GetPluginInfo()
contract.IgnoreError(err) // nonfatal error
version = info.Version
}
return schemaBytes, version, nil
}
pluginInfo, err := l.host.ResolvePlugin(apitype.ResourcePlugin, pkg, version)
if err != nil {
Add an envar to disable automatic provider installation (#14083) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> This is primarily for the providers team to enable during builds so they can have more confidence about reproducibility of builds (especially examples conversion), but I imagine some customers would enable this as well. Fixes #14086 ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [ ] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-10-03 15:35:23 +00:00
// Try and install the plugin if it was missing and try again, unless auto plugin installs are turned off.
if env.DisableAutomaticPluginAcquisition.Value() {
return nil, nil, err
}
var missingError *workspace.MissingError
if errors.As(err, &missingError) {
spec := workspace.PluginSpec{
Kind: apitype.ResourcePlugin,
Name: pkg,
Version: version,
}
log := func(sev diag.Severity, msg string) {
l.host.Log(sev, "", msg, 0)
}
_, err = pkgWorkspace.InstallPlugin(spec, log)
if err != nil {
return nil, nil, err
}
pluginInfo, err = l.host.ResolvePlugin(apitype.ResourcePlugin, pkg, version)
if err != nil {
return nil, version, err
}
} else {
return nil, nil, err
}
}
contract.Assertf(pluginInfo != nil, "loading pkg %q: pluginInfo was unexpectedly nil", pkg)
if version == nil {
version = pluginInfo.Version
}
if pluginInfo.SchemaPath != "" && version != nil {
schemaBytes, ok := l.loadCachedSchemaBytes(pkg, pluginInfo.SchemaPath, pluginInfo.SchemaTime)
if ok {
return schemaBytes, nil, nil
}
}
schemaBytes, provider, err := l.loadPluginSchemaBytes(pkg, version)
if err != nil {
return nil, nil, fmt.Errorf("Error loading schema from plugin: %w", err)
}
if pluginInfo.SchemaPath != "" {
err = atomic.WriteFile(pluginInfo.SchemaPath, bytes.NewReader(schemaBytes))
if err != nil {
return nil, nil, fmt.Errorf("Error writing schema from plugin to cache: %w", err)
}
}
if version == nil {
info, _ := provider.GetPluginInfo() // nonfatal error
version = info.Version
}
return schemaBytes, version, nil
}
func (l *pluginLoader) loadPluginSchemaBytes(pkg string, version *semver.Version) ([]byte, plugin.Provider, error) {
provider, err := l.host.Provider(tokens.Package(pkg), version)
if err != nil {
return nil, nil, err
}
contract.Assertf(provider != nil, "unexpected nil provider for %s@%v", pkg, version)
schemaFormatVersion := 0
schemaBytes, err := provider.GetSchema(schemaFormatVersion)
if err != nil {
return nil, nil, err
}
return schemaBytes, provider, nil
}
var mmapedFiles = make(map[string]mmap.MMap)
func (l *pluginLoader) loadCachedSchemaBytes(pkg string, path string, schemaTime time.Time) ([]byte, bool) {
if l.cacheOptions.disableFileCache {
return nil, false
}
if schemaMmap, ok := mmapedFiles[path]; ok {
return schemaMmap, true
}
success := false
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
schemaFile, err := os.OpenFile(path, os.O_RDONLY, 0o644)
defer func() {
if !success {
schemaFile.Close()
}
}()
if err != nil {
return nil, success
}
stat, err := schemaFile.Stat()
if err != nil {
return nil, success
}
cachedAt := stat.ModTime()
if schemaTime.After(cachedAt) {
return nil, success
}
if l.cacheOptions.disableMmap {
data, err := io.ReadAll(schemaFile)
if err != nil {
return nil, success
}
success = true
return data, success
}
schemaMmap, err := mmap.Map(schemaFile, mmap.RDONLY, 0)
if err != nil {
return nil, success
}
success = true
return schemaMmap, success
}