2021-12-10 21:23:54 +00:00
|
|
|
// Copyright 2016-2021, Pulumi Corporation.
|
2018-05-22 19:43:36 +00:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
|
|
|
|
package workspace
|
|
|
|
|
|
|
|
import (
|
2020-11-21 16:02:05 +00:00
|
|
|
"bytes"
|
2022-06-09 21:57:56 +00:00
|
|
|
"context"
|
2022-08-30 09:44:56 +00:00
|
|
|
"crypto/sha256"
|
2022-02-17 21:20:29 +00:00
|
|
|
"encoding/json"
|
2023-01-19 20:22:54 +00:00
|
|
|
"errors"
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
"fmt"
|
2022-08-30 09:44:56 +00:00
|
|
|
"hash"
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
"io"
|
2022-07-15 04:32:40 +00:00
|
|
|
"io/fs"
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
"net/http"
|
2020-05-29 21:59:25 +00:00
|
|
|
"net/url"
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
"os"
|
2018-02-06 15:20:04 +00:00
|
|
|
"os/exec"
|
2019-06-30 23:34:39 +00:00
|
|
|
"path"
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
"path/filepath"
|
|
|
|
"regexp"
|
2018-02-17 19:42:27 +00:00
|
|
|
"runtime"
|
2019-03-01 23:42:38 +00:00
|
|
|
"sort"
|
2023-03-02 13:34:36 +00:00
|
|
|
"strconv"
|
2020-05-25 16:16:57 +00:00
|
|
|
"strings"
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/blang/semver"
|
2019-08-21 12:22:41 +00:00
|
|
|
"github.com/cheggaaa/pb"
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
"github.com/djherbis/times"
|
2018-02-06 15:20:04 +00:00
|
|
|
|
2023-08-07 12:15:57 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
|
2021-03-17 13:20:05 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors"
|
2022-12-14 14:18:13 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/env"
|
2021-03-17 13:20:05 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/archive"
|
2021-12-10 21:23:54 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
|
2021-03-17 13:20:05 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/fsutil"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/httputil"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
|
2023-03-20 20:20:38 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/retry"
|
2021-03-17 13:20:05 +00:00
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/version"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/nodejs/npm"
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/python"
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
)
|
|
|
|
|
Look for language plugins next to `pulumi` when probing
When launching plugins today, `pulumi` looks in two places:
1. It looks to see if the plugin in on the $PATH and if so, uses
it. This makes it easy to force a specific version of a resource
provider to be used and is what happens at development time (since
resource providers make their way onto $PATH via GOBIN).
2. If the above fails, it looks in the "plugin cache" in
`~/.pulumi/plugins`. This is the location that `pulumi plugin
install` places plugins.
Unlike resource provider plugins, we don't yet deliver language
plugins via `pulumi plugin install` so the language provider plugins
must be on the `$PATH` to be found. This is okay, because when we ship
the SDK, we include the executables next to `pulumi` itself.
However, if a user chooses to not put `pulumi` on their $PATH, or they
do but it is a symlink to the real `pulumi` binary installed
somewhere, we'd fail to find the language plugins, since they would
not be on the `$PATH`
To address this, when probing for language plugins, also consider
binaries next to the currently running `pulumi` process.
Fixes #1956
2018-11-19 23:07:01 +00:00
|
|
|
const (
|
|
|
|
windowsGOOS = "windows"
|
|
|
|
)
|
|
|
|
|
2023-03-03 16:36:39 +00:00
|
|
|
var enableLegacyPluginBehavior = os.Getenv("PULUMI_ENABLE_LEGACY_PLUGIN_SEARCH") != ""
|
2019-03-01 23:42:38 +00:00
|
|
|
|
2022-01-21 17:04:10 +00:00
|
|
|
// pluginDownloadURLOverrides is a variable instead of a constant so it can be set using the `-X` `ldflag` at build
|
|
|
|
// time, if necessary. When non-empty, it's parsed into `pluginDownloadURLOverridesParsed` in `init()`. The expected
|
|
|
|
// format is `regexp=URL`, and multiple pairs can be specified separated by commas, e.g. `regexp1=URL1,regexp2=URL2`.
|
|
|
|
//
|
2022-02-06 23:28:35 +00:00
|
|
|
// For example, when set to "^foo.*=https://foo,^bar.*=https://bar", plugin names that start with "foo" will use
|
2022-01-21 17:04:10 +00:00
|
|
|
// https://foo as the download URL and names that start with "bar" will use https://bar.
|
|
|
|
var pluginDownloadURLOverrides string
|
|
|
|
|
|
|
|
// pluginDownloadURLOverridesParsed is the parsed array from `pluginDownloadURLOverrides`.
|
|
|
|
var pluginDownloadURLOverridesParsed pluginDownloadOverrideArray
|
|
|
|
|
|
|
|
// pluginDownloadURLOverride represents a plugin download URL override, parsed from `pluginDownloadURLOverrides`.
|
|
|
|
type pluginDownloadURLOverride struct {
|
|
|
|
reg *regexp.Regexp // The regex used to match against the plugin's name.
|
|
|
|
url string // The URL to use for the matched plugin.
|
|
|
|
}
|
|
|
|
|
|
|
|
// pluginDownloadOverrideArray represents an array of overrides.
|
|
|
|
type pluginDownloadOverrideArray []pluginDownloadURLOverride
|
|
|
|
|
|
|
|
// get returns the URL and true if name matches an override's regular expression,
|
|
|
|
// otherwise an empty string and false.
|
|
|
|
func (overrides pluginDownloadOverrideArray) get(name string) (string, bool) {
|
|
|
|
for _, override := range overrides {
|
|
|
|
if override.reg.MatchString(name) {
|
|
|
|
return override.url, true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return "", false
|
|
|
|
}
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
var err error
|
|
|
|
if pluginDownloadURLOverridesParsed, err = parsePluginDownloadURLOverrides(pluginDownloadURLOverrides); err != nil {
|
|
|
|
panic(fmt.Errorf("error parsing `pluginDownloadURLOverrides`: %w", err))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// parsePluginDownloadURLOverrides parses an overrides string with the expected format `regexp1=URL1,regexp2=URL2`.
|
|
|
|
func parsePluginDownloadURLOverrides(overrides string) (pluginDownloadOverrideArray, error) {
|
2023-03-03 16:36:39 +00:00
|
|
|
splits := strings.Split(overrides, ",")
|
|
|
|
result := make(pluginDownloadOverrideArray, 0, len(splits))
|
2022-01-21 17:04:10 +00:00
|
|
|
if overrides == "" {
|
|
|
|
return result, nil
|
|
|
|
}
|
2023-01-11 20:52:51 +00:00
|
|
|
for _, pair := range splits {
|
2022-01-21 17:04:10 +00:00
|
|
|
split := strings.Split(pair, "=")
|
|
|
|
if len(split) != 2 || split[0] == "" || split[1] == "" {
|
|
|
|
return nil, fmt.Errorf("expected format to be \"regexp1=URL1,regexp2=URL2\"; got %q", overrides)
|
|
|
|
}
|
|
|
|
reg, err := regexp.Compile(split[0])
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
result = append(result, pluginDownloadURLOverride{
|
|
|
|
reg: reg,
|
|
|
|
url: split[1],
|
|
|
|
})
|
|
|
|
}
|
|
|
|
return result, nil
|
|
|
|
}
|
|
|
|
|
2023-03-15 15:24:39 +00:00
|
|
|
// MissingError is returned by functions that attempt to load plugins if a plugin can't be located.
|
|
|
|
type MissingError struct {
|
|
|
|
// Kind of the plugin that couldn't be found.
|
|
|
|
kind PluginKind
|
|
|
|
// Name of the plugin that couldn't be found.
|
|
|
|
name string
|
|
|
|
// Optional version of the plugin that couldn't be found.
|
|
|
|
version *semver.Version
|
|
|
|
// includeAmbient is true if we search $PATH for this plugin
|
|
|
|
includeAmbient bool
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewMissingError allocates a new error indicating the given plugin info was not found.
|
|
|
|
func NewMissingError(kind PluginKind, name string, version *semver.Version, includeAmbient bool) error {
|
|
|
|
return &MissingError{
|
|
|
|
kind: kind,
|
|
|
|
name: name,
|
|
|
|
version: version,
|
|
|
|
includeAmbient: includeAmbient,
|
|
|
|
}
|
2022-09-08 04:41:23 +00:00
|
|
|
}
|
|
|
|
|
2023-03-15 15:24:39 +00:00
|
|
|
func (err *MissingError) Error() string {
|
|
|
|
includePath := ""
|
|
|
|
if err.includeAmbient {
|
|
|
|
includePath = " or on your $PATH"
|
2022-09-08 04:41:23 +00:00
|
|
|
}
|
|
|
|
|
2023-03-15 15:24:39 +00:00
|
|
|
if err.version != nil {
|
|
|
|
return fmt.Sprintf("no %[1]s plugin 'pulumi-%[1]s-%[2]s' found in the workspace at version v%[3]s%[4]s",
|
|
|
|
err.kind, err.name, err.version, includePath)
|
|
|
|
}
|
|
|
|
|
|
|
|
return fmt.Sprintf("no %[1]s plugin 'pulumi-%[1]s-%[2]s' found in the workspace%[3]s",
|
|
|
|
err.kind, err.name, includePath)
|
2022-09-08 04:41:23 +00:00
|
|
|
}
|
|
|
|
|
2022-03-03 09:22:21 +00:00
|
|
|
// PluginSource deals with downloading a specific version of a plugin, or looking up the latest version of it.
|
|
|
|
type PluginSource interface {
|
|
|
|
// Download fetches an io.ReadCloser for this plugin and also returns the size of the response (if known).
|
|
|
|
Download(
|
|
|
|
version semver.Version, opSy string, arch string,
|
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error)) (io.ReadCloser, int64, error)
|
|
|
|
// GetLatestVersion tries to find the latest version for this plugin. This is currently only supported for
|
|
|
|
// plugins we can get from github releases.
|
|
|
|
GetLatestVersion(getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error)) (*semver.Version, error)
|
|
|
|
}
|
|
|
|
|
2023-03-04 09:13:30 +00:00
|
|
|
// standardAssetName returns the standard name for the asset that contains the given plugin.
|
|
|
|
func standardAssetName(name string, kind PluginKind, version semver.Version, opSy, arch string) string {
|
|
|
|
return fmt.Sprintf("pulumi-%s-%s-v%s-%s-%s.tar.gz", kind, name, version, opSy, arch)
|
|
|
|
}
|
|
|
|
|
2022-03-03 09:22:21 +00:00
|
|
|
// getPulumiSource can download a plugin from get.pulumi.com
|
|
|
|
type getPulumiSource struct {
|
|
|
|
name string
|
|
|
|
kind PluginKind
|
|
|
|
}
|
|
|
|
|
|
|
|
func newGetPulumiSource(name string, kind PluginKind) *getPulumiSource {
|
|
|
|
return &getPulumiSource{name: name, kind: kind}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (source *getPulumiSource) GetLatestVersion(
|
2023-03-03 16:36:39 +00:00
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
|
|
|
) (*semver.Version, error) {
|
2022-03-03 09:22:21 +00:00
|
|
|
return nil, errors.New("GetLatestVersion is not supported for plugins from get.pulumi.com")
|
|
|
|
}
|
|
|
|
|
|
|
|
func (source *getPulumiSource) Download(
|
|
|
|
version semver.Version, opSy string, arch string,
|
2023-03-03 16:36:39 +00:00
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
|
|
|
) (io.ReadCloser, int64, error) {
|
2022-03-03 09:22:21 +00:00
|
|
|
serverURL := "https://get.pulumi.com/releases/plugins"
|
|
|
|
logging.V(1).Infof("%s downloading from %s", source.name, serverURL)
|
|
|
|
endpoint := fmt.Sprintf("%s/%s",
|
|
|
|
serverURL,
|
2023-03-04 09:13:30 +00:00
|
|
|
url.QueryEscape(standardAssetName(source.name, source.kind, version, opSy, arch)))
|
2022-03-03 09:22:21 +00:00
|
|
|
|
2022-03-14 21:14:36 +00:00
|
|
|
req, err := buildHTTPRequest(endpoint, "")
|
2022-03-03 09:22:21 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
|
|
|
return getHTTPResponse(req)
|
|
|
|
}
|
|
|
|
|
2023-02-11 11:05:06 +00:00
|
|
|
// gitlabSource can download a plugin from gitlab releases.
|
|
|
|
type gitlabSource struct {
|
|
|
|
host string
|
|
|
|
project string
|
|
|
|
name string
|
|
|
|
kind PluginKind
|
|
|
|
|
|
|
|
token string
|
|
|
|
}
|
|
|
|
|
|
|
|
// Creates a new GitLab source from a gitlab://<host>/<project_id> url.
|
|
|
|
// Uses the GITLAB_TOKEN environment variable for authentication if it's set.
|
|
|
|
func newGitlabSource(url *url.URL, name string, kind PluginKind) (*gitlabSource, error) {
|
2023-02-17 01:23:09 +00:00
|
|
|
contract.Requiref(url.Scheme == "gitlab", "url", `scheme must be "gitlab", was %q`, url.Scheme)
|
2023-02-11 11:05:06 +00:00
|
|
|
|
|
|
|
host := url.Host
|
|
|
|
if host == "" {
|
|
|
|
return nil, fmt.Errorf("gitlab:// url must have a host part, was: %s", url)
|
|
|
|
}
|
|
|
|
|
|
|
|
project := strings.Trim(url.Path, "/")
|
|
|
|
if project == "" || strings.Contains(project, "/") {
|
|
|
|
return nil, fmt.Errorf(
|
|
|
|
"gitlab:// url must have the format <host>/<project>, was: %s",
|
|
|
|
url)
|
|
|
|
}
|
|
|
|
|
|
|
|
return &gitlabSource{
|
|
|
|
host: host,
|
|
|
|
project: project,
|
|
|
|
name: name,
|
|
|
|
kind: kind,
|
|
|
|
|
|
|
|
token: os.Getenv("GITLAB_TOKEN"),
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (source *gitlabSource) newHTTPRequest(url, accept string) (*http.Request, error) {
|
2023-03-09 09:36:55 +00:00
|
|
|
var authorization string
|
|
|
|
if source.token != "" {
|
2023-12-12 12:19:42 +00:00
|
|
|
authorization = "Bearer " + source.token
|
2023-03-09 09:36:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
req, err := buildHTTPRequest(url, authorization)
|
2023-02-11 11:05:06 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
req.Header.Set("Accept", accept)
|
|
|
|
return req, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (source *gitlabSource) GetLatestVersion(
|
2023-03-03 16:36:39 +00:00
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
|
|
|
) (*semver.Version, error) {
|
2023-02-11 11:05:06 +00:00
|
|
|
releaseURL := fmt.Sprintf(
|
|
|
|
"https://%s/api/v4/projects/%s/releases/permalink/latest",
|
|
|
|
source.host, source.project)
|
|
|
|
logging.V(9).Infof("plugin GitLab releases url: %s", releaseURL)
|
|
|
|
req, err := source.newHTTPRequest(releaseURL, "application/json")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
resp, length, err := getHTTPResponse(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer contract.IgnoreClose(resp)
|
|
|
|
|
|
|
|
var release struct {
|
|
|
|
TagName string `json:"tag_name"`
|
|
|
|
}
|
|
|
|
if err = json.NewDecoder(resp).Decode(&release); err != nil {
|
|
|
|
return nil, fmt.Errorf("cannot decode gitlab response len(%d): %w", length, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
parsedVersion, err := semver.ParseTolerant(release.TagName)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("invalid plugin version %s: %w", release.TagName, err)
|
|
|
|
}
|
|
|
|
return &parsedVersion, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (source *gitlabSource) Download(
|
|
|
|
version semver.Version, opSy string, arch string,
|
2023-03-03 16:36:39 +00:00
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
|
|
|
) (io.ReadCloser, int64, error) {
|
2023-03-04 09:13:30 +00:00
|
|
|
assetName := standardAssetName(source.name, source.kind, version, opSy, arch)
|
2023-02-11 11:05:06 +00:00
|
|
|
|
|
|
|
assetURL := fmt.Sprintf(
|
|
|
|
"https://%s/api/v4/projects/%s/releases/v%s/downloads/%s",
|
|
|
|
source.host, source.project, version, assetName)
|
|
|
|
logging.V(1).Infof("%s downloading from %s", source.name, assetURL)
|
|
|
|
|
|
|
|
req, err := source.newHTTPRequest(assetURL, "application/octet-stream")
|
|
|
|
if err != nil {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
|
|
|
return getHTTPResponse(req)
|
|
|
|
}
|
|
|
|
|
2022-03-03 09:22:21 +00:00
|
|
|
// githubSource can download a plugin from github releases
|
|
|
|
type githubSource struct {
|
2022-06-29 19:15:01 +00:00
|
|
|
host string
|
2022-03-03 09:22:21 +00:00
|
|
|
organization string
|
2022-06-29 19:15:01 +00:00
|
|
|
repository string
|
2022-03-03 09:22:21 +00:00
|
|
|
name string
|
|
|
|
kind PluginKind
|
|
|
|
|
2022-03-14 21:14:36 +00:00
|
|
|
token string
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
|
2022-03-14 21:14:36 +00:00
|
|
|
// Creates a new github source adding authentication data in the environment, if it exists
|
2022-06-29 19:15:01 +00:00
|
|
|
func newGithubSource(url *url.URL, name string, kind PluginKind) (*githubSource, error) {
|
2023-02-15 01:06:56 +00:00
|
|
|
contract.Requiref(url.Scheme == "github", "url", `scheme must be "github", was %q`, url.Scheme)
|
2022-03-03 09:22:21 +00:00
|
|
|
|
2022-03-14 21:14:36 +00:00
|
|
|
// 14-03-2022 we stopped looking at GITHUB_PERSONAL_ACCESS_TOKEN and sending basic auth for github and
|
|
|
|
// instead just look at GITHUB_TOKEN and send in a header. Given GITHUB_PERSONAL_ACCESS_TOKEN was an
|
|
|
|
// envvar we made up we check to see if it's set here and log a warning. This can be removed after a few
|
|
|
|
// releases.
|
|
|
|
if os.Getenv("GITHUB_PERSONAL_ACCESS_TOKEN") != "" {
|
|
|
|
logging.Warningf("GITHUB_PERSONAL_ACCESS_TOKEN is no longer used for Github authentication, set GITHUB_TOKEN instead")
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
|
2022-06-29 19:15:01 +00:00
|
|
|
host := url.Host
|
|
|
|
parts := strings.Split(strings.Trim(url.Path, "/"), "/")
|
|
|
|
|
|
|
|
if host == "" {
|
2023-02-11 11:05:06 +00:00
|
|
|
return nil, fmt.Errorf("github:// url must have a host part, was: %s", url)
|
2022-06-29 19:15:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if len(parts) != 1 && len(parts) != 2 {
|
|
|
|
return nil, fmt.Errorf(
|
|
|
|
"github:// url must have the format <host>/<organization>[/<repository>], was: %s",
|
2023-02-11 11:05:06 +00:00
|
|
|
url)
|
2022-06-29 19:15:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
organization := parts[0]
|
2023-02-11 11:05:06 +00:00
|
|
|
if organization == "" {
|
|
|
|
return nil, fmt.Errorf(
|
|
|
|
"github:// url must have the format <host>/<organization>[/<repository>], was: %s",
|
|
|
|
url)
|
|
|
|
}
|
|
|
|
|
2022-06-29 19:15:01 +00:00
|
|
|
repository := "pulumi-" + name
|
2023-03-04 09:13:30 +00:00
|
|
|
if kind == ConverterPlugin {
|
|
|
|
// Converter plugins are expected at a different repo path, e.g.
|
|
|
|
// github.com/pulumi/pulumi-converter-aws rather than github.com/pulumi/pulumi-aws which would clash
|
|
|
|
// with the providers of the same name.
|
|
|
|
repository = "pulumi-converter-" + name
|
2023-03-13 18:29:23 +00:00
|
|
|
if name == "yaml" {
|
|
|
|
// We special case the yaml converter plugin to be in the pulumi-yaml repo. It's not ideal but its
|
|
|
|
// to have this hardcoded here than having to deal with two repos for YAML, and long term this
|
|
|
|
// should go away and be replaced with a registry lookup.
|
|
|
|
repository = "pulumi-yaml"
|
|
|
|
}
|
2023-03-04 09:13:30 +00:00
|
|
|
}
|
2022-06-29 19:15:01 +00:00
|
|
|
if len(parts) == 2 {
|
|
|
|
repository = parts[1]
|
|
|
|
}
|
|
|
|
|
2022-03-14 21:14:36 +00:00
|
|
|
return &githubSource{
|
2022-06-29 19:15:01 +00:00
|
|
|
host: host,
|
2022-03-03 09:22:21 +00:00
|
|
|
organization: organization,
|
2022-06-29 19:15:01 +00:00
|
|
|
repository: repository,
|
2022-03-03 09:22:21 +00:00
|
|
|
name: name,
|
|
|
|
kind: kind,
|
|
|
|
|
2022-03-14 21:14:36 +00:00
|
|
|
token: os.Getenv("GITHUB_TOKEN"),
|
2022-06-29 19:15:01 +00:00
|
|
|
}, nil
|
2022-03-14 21:14:36 +00:00
|
|
|
}
|
2022-03-03 09:22:21 +00:00
|
|
|
|
2023-02-11 11:05:06 +00:00
|
|
|
func (source *githubSource) newHTTPRequest(url, accept string) (*http.Request, error) {
|
2023-03-09 09:36:55 +00:00
|
|
|
var authorization string
|
|
|
|
if source.token != "" {
|
2023-12-12 12:19:42 +00:00
|
|
|
authorization = "token " + source.token
|
2023-03-09 09:36:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
req, err := buildHTTPRequest(url, authorization)
|
2023-02-11 11:05:06 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
req.Header.Set("Accept", accept)
|
|
|
|
return req, nil
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
|
2023-03-02 13:34:36 +00:00
|
|
|
func (source *githubSource) getHTTPResponse(
|
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
2023-03-03 16:36:39 +00:00
|
|
|
req *http.Request,
|
|
|
|
) (io.ReadCloser, int64, error) {
|
2023-03-02 13:34:36 +00:00
|
|
|
resp, length, err := getHTTPResponse(req)
|
|
|
|
if err == nil {
|
|
|
|
return resp, length, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Wrap 403 rate limit errors with a more helpful message.
|
|
|
|
var downErr *downloadError
|
|
|
|
if !errors.As(err, &downErr) || downErr.code != 403 {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// This is a rate limiting error only if x-ratelimit-remaining is 0.
|
|
|
|
// https://docs.github.com/en/rest/overview/resources-in-the-rest-api?apiVersion=2022-11-28#exceeding-the-rate-limit
|
|
|
|
if downErr.header.Get("x-ratelimit-remaining") != "0" {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
|
|
|
|
|
|
|
tryAgain := "."
|
|
|
|
if reset, err := strconv.ParseInt(downErr.header.Get("x-ratelimit-reset"), 10, 64); err == nil {
|
|
|
|
delay := time.Until(time.Unix(reset, 0).UTC())
|
|
|
|
tryAgain = fmt.Sprintf(", try again in %s.", delay)
|
|
|
|
}
|
|
|
|
|
|
|
|
addAuth := ""
|
|
|
|
if source.token == "" {
|
|
|
|
addAuth = " You can set GITHUB_TOKEN to make an authenticated request with a higher rate limit."
|
|
|
|
}
|
|
|
|
|
2023-03-17 14:33:13 +00:00
|
|
|
logging.Errorf("GitHub rate limit exceeded for %s%s%s", req.URL, tryAgain, addAuth)
|
2023-03-02 13:34:36 +00:00
|
|
|
return nil, -1, fmt.Errorf("rate limit exceeded: %w", err)
|
|
|
|
}
|
|
|
|
|
2022-03-03 09:22:21 +00:00
|
|
|
func (source *githubSource) GetLatestVersion(
|
2023-03-03 16:36:39 +00:00
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
|
|
|
) (*semver.Version, error) {
|
2022-03-03 09:22:21 +00:00
|
|
|
releaseURL := fmt.Sprintf(
|
2022-06-29 19:15:01 +00:00
|
|
|
"https://%s/repos/%s/%s/releases/latest",
|
|
|
|
source.host, source.organization, source.repository)
|
2022-03-03 09:22:21 +00:00
|
|
|
logging.V(9).Infof("plugin GitHub releases url: %s", releaseURL)
|
2023-02-11 11:05:06 +00:00
|
|
|
req, err := source.newHTTPRequest(releaseURL, "application/json")
|
2022-03-03 09:22:21 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-03-02 13:34:36 +00:00
|
|
|
resp, length, err := source.getHTTPResponse(getHTTPResponse, req)
|
2022-03-03 09:22:21 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-02-11 11:05:06 +00:00
|
|
|
defer contract.IgnoreClose(resp)
|
|
|
|
|
|
|
|
var release struct {
|
2022-03-03 09:22:21 +00:00
|
|
|
TagName string `json:"tag_name"`
|
|
|
|
}
|
2023-02-11 11:05:06 +00:00
|
|
|
if err = json.NewDecoder(resp).Decode(&release); err != nil {
|
|
|
|
return nil, fmt.Errorf("cannot decode github response len(%d): %w", length, err)
|
|
|
|
}
|
|
|
|
|
2022-03-03 09:22:21 +00:00
|
|
|
parsedVersion, err := semver.ParseTolerant(release.TagName)
|
|
|
|
if err != nil {
|
2023-02-11 11:05:06 +00:00
|
|
|
return nil, fmt.Errorf("invalid plugin version %s: %w", release.TagName, err)
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
return &parsedVersion, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (source *githubSource) Download(
|
|
|
|
version semver.Version, opSy string, arch string,
|
2023-03-03 16:36:39 +00:00
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
|
|
|
) (io.ReadCloser, int64, error) {
|
2022-03-03 09:22:21 +00:00
|
|
|
releaseURL := fmt.Sprintf(
|
2022-06-29 19:15:01 +00:00
|
|
|
"https://%s/repos/%s/%s/releases/tags/v%s",
|
2023-02-11 11:05:06 +00:00
|
|
|
source.host, source.organization, source.repository, version)
|
2022-03-03 09:22:21 +00:00
|
|
|
logging.V(9).Infof("plugin GitHub releases url: %s", releaseURL)
|
|
|
|
|
2023-02-11 11:05:06 +00:00
|
|
|
req, err := source.newHTTPRequest(releaseURL, "application/json")
|
2022-03-03 09:22:21 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
2023-03-02 13:34:36 +00:00
|
|
|
resp, length, err := source.getHTTPResponse(getHTTPResponse, req)
|
2022-03-03 09:22:21 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
2023-02-11 11:05:06 +00:00
|
|
|
defer contract.IgnoreClose(resp)
|
|
|
|
|
|
|
|
var release struct {
|
2022-03-03 09:22:21 +00:00
|
|
|
Assets []struct {
|
|
|
|
Name string `json:"name"`
|
|
|
|
URL string `json:"url"`
|
|
|
|
} `json:"assets"`
|
|
|
|
}
|
2023-02-11 11:05:06 +00:00
|
|
|
if err = json.NewDecoder(resp).Decode(&release); err != nil {
|
|
|
|
return nil, -1, fmt.Errorf("cannot decode github response len(%d): %w", length, err)
|
|
|
|
}
|
|
|
|
|
2023-03-04 09:13:30 +00:00
|
|
|
assetName := standardAssetName(source.name, source.kind, version, opSy, arch)
|
2022-03-03 09:22:21 +00:00
|
|
|
assetURL := ""
|
|
|
|
for _, asset := range release.Assets {
|
|
|
|
if asset.Name == assetName {
|
|
|
|
assetURL = asset.URL
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if assetURL == "" {
|
2023-02-11 11:05:06 +00:00
|
|
|
logging.V(9).Infof("github response: %v", release)
|
2022-03-03 09:22:21 +00:00
|
|
|
logging.V(9).Infof("plugin asset '%s' not found", assetName)
|
2023-01-19 20:22:54 +00:00
|
|
|
return nil, -1, fmt.Errorf("plugin asset '%s' not found", assetName)
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
logging.V(1).Infof("%s downloading from %s", source.name, assetURL)
|
|
|
|
|
2023-02-11 11:05:06 +00:00
|
|
|
req, err = source.newHTTPRequest(assetURL, "application/octet-stream")
|
2022-03-03 09:22:21 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
2023-03-02 13:34:36 +00:00
|
|
|
return source.getHTTPResponse(getHTTPResponse, req)
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
|
2023-02-15 19:48:02 +00:00
|
|
|
// httpSource can download a plugin from a given http url, it doesn't support GetLatestVersion
|
|
|
|
type httpSource struct {
|
|
|
|
name string
|
|
|
|
kind PluginKind
|
|
|
|
url string
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
|
2023-02-15 19:48:02 +00:00
|
|
|
func newHTTPSource(name string, kind PluginKind, url *url.URL) *httpSource {
|
|
|
|
contract.Requiref(
|
|
|
|
url.Scheme == "http" || url.Scheme == "https",
|
|
|
|
"url", `scheme must be "http" or "https", was %q`, url.Scheme)
|
|
|
|
|
|
|
|
return &httpSource{
|
|
|
|
name: name,
|
|
|
|
kind: kind,
|
|
|
|
url: url.String(),
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-02-15 19:48:02 +00:00
|
|
|
func (source *httpSource) GetLatestVersion(
|
2023-03-03 16:36:39 +00:00
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
|
|
|
) (*semver.Version, error) {
|
2023-02-15 19:48:02 +00:00
|
|
|
return nil, errors.New("GetLatestVersion is not supported for plugins from http sources")
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
|
2023-10-30 09:29:12 +00:00
|
|
|
func interpolateURL(serverURL string, name string, version semver.Version, os, arch string) string {
|
|
|
|
// Expectation is the URL is already escaped, so we need to escape the {}'s in the replacement strings.
|
2023-07-10 09:16:51 +00:00
|
|
|
replacer := strings.NewReplacer(
|
2023-10-30 09:29:12 +00:00
|
|
|
"$%7BNAME%7D", url.QueryEscape(name),
|
2023-07-10 09:16:51 +00:00
|
|
|
"$%7BVERSION%7D", url.QueryEscape(version.String()),
|
|
|
|
"$%7BOS%7D", url.QueryEscape(os),
|
|
|
|
"$%7BARCH%7D", url.QueryEscape(arch))
|
|
|
|
return replacer.Replace(serverURL)
|
|
|
|
}
|
|
|
|
|
2023-02-15 19:48:02 +00:00
|
|
|
func (source *httpSource) Download(
|
2022-03-03 09:22:21 +00:00
|
|
|
version semver.Version, opSy string, arch string,
|
2023-03-03 16:36:39 +00:00
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
|
|
|
) (io.ReadCloser, int64, error) {
|
2023-10-30 09:29:12 +00:00
|
|
|
serverURL := interpolateURL(source.url, source.name, version, opSy, arch)
|
2022-03-03 09:22:21 +00:00
|
|
|
serverURL = strings.TrimSuffix(serverURL, "/")
|
|
|
|
logging.V(1).Infof("%s downloading from %s", source.name, serverURL)
|
2023-02-15 19:48:02 +00:00
|
|
|
|
2022-03-03 09:22:21 +00:00
|
|
|
endpoint := fmt.Sprintf("%s/%s",
|
|
|
|
serverURL,
|
2023-02-11 11:05:06 +00:00
|
|
|
url.QueryEscape(fmt.Sprintf("pulumi-%s-%s-v%s-%s-%s.tar.gz", source.kind, source.name, version, opSy, arch)))
|
2022-03-03 09:22:21 +00:00
|
|
|
|
2022-03-14 21:14:36 +00:00
|
|
|
req, err := buildHTTPRequest(endpoint, "")
|
2022-03-03 09:22:21 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
|
|
|
return getHTTPResponse(req)
|
|
|
|
}
|
|
|
|
|
2023-02-15 19:48:02 +00:00
|
|
|
// fallbackSource handles our current default logic of trying the pulumi public github then get.pulumi.com.
|
2022-03-03 09:22:21 +00:00
|
|
|
type fallbackSource struct {
|
|
|
|
name string
|
|
|
|
kind PluginKind
|
|
|
|
}
|
|
|
|
|
|
|
|
func newFallbackSource(name string, kind PluginKind) *fallbackSource {
|
|
|
|
return &fallbackSource{
|
|
|
|
name: name,
|
|
|
|
kind: kind,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-06-29 19:15:01 +00:00
|
|
|
func urlMustParse(rawURL string) *url.URL {
|
|
|
|
url, err := url.Parse(rawURL)
|
2023-02-15 01:06:56 +00:00
|
|
|
contract.AssertNoErrorf(err, "url.Parse(%q)", rawURL)
|
2022-06-29 19:15:01 +00:00
|
|
|
return url
|
|
|
|
}
|
|
|
|
|
2022-03-03 09:22:21 +00:00
|
|
|
func (source *fallbackSource) GetLatestVersion(
|
2023-03-03 16:36:39 +00:00
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
|
|
|
) (*semver.Version, error) {
|
2022-10-01 22:55:36 +00:00
|
|
|
// Try and get this package from our public pulumi github
|
2022-06-29 19:15:01 +00:00
|
|
|
public, err := newGithubSource(urlMustParse("github://api.github.com/pulumi"), source.name, source.kind)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-03-03 09:22:21 +00:00
|
|
|
version, err := public.GetLatestVersion(getHTTPResponse)
|
2022-10-01 22:55:36 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
|
2022-10-01 22:55:36 +00:00
|
|
|
return version, nil
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (source *fallbackSource) Download(
|
|
|
|
version semver.Version, opSy string, arch string,
|
2023-03-03 16:36:39 +00:00
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
|
|
|
) (io.ReadCloser, int64, error) {
|
2022-03-03 09:22:21 +00:00
|
|
|
// Try and get this package from public pulumi github
|
2022-06-29 19:15:01 +00:00
|
|
|
public, err := newGithubSource(urlMustParse("github://api.github.com/pulumi"), source.name, source.kind)
|
|
|
|
if err != nil {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
2022-03-03 09:22:21 +00:00
|
|
|
resp, length, err := public.Download(version, opSy, arch, getHTTPResponse)
|
|
|
|
if err == nil {
|
|
|
|
return resp, length, nil
|
|
|
|
}
|
2023-04-12 11:10:16 +00:00
|
|
|
logging.Infof("Failed to download from GitHub, falling back to get.pulumi.com: %v", err)
|
2022-03-03 09:22:21 +00:00
|
|
|
|
|
|
|
// Fallback to get.pulumi.com
|
|
|
|
pulumi := newGetPulumiSource(source.name, source.kind)
|
|
|
|
return pulumi.Download(version, opSy, arch, getHTTPResponse)
|
|
|
|
}
|
|
|
|
|
2022-08-30 15:11:36 +00:00
|
|
|
type checksumError struct {
|
|
|
|
expected []byte
|
|
|
|
actual []byte
|
|
|
|
}
|
|
|
|
|
|
|
|
func (err *checksumError) Error() string {
|
|
|
|
return fmt.Sprintf("invalid checksum, expected %x, actual %x", err.expected, err.actual)
|
|
|
|
}
|
|
|
|
|
2022-08-30 09:44:56 +00:00
|
|
|
// checksumSource will validate that the archive downloaded from the inner source matches a checksum
|
|
|
|
type checksumSource struct {
|
|
|
|
source PluginSource
|
|
|
|
checksum map[string][]byte
|
|
|
|
}
|
|
|
|
|
|
|
|
func newChecksumSource(source PluginSource, checksum map[string][]byte) *checksumSource {
|
|
|
|
return &checksumSource{
|
|
|
|
source: source,
|
|
|
|
checksum: checksum,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (source *checksumSource) GetLatestVersion(
|
2023-03-03 16:36:39 +00:00
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
|
|
|
) (*semver.Version, error) {
|
2022-08-30 09:44:56 +00:00
|
|
|
return source.source.GetLatestVersion(getHTTPResponse)
|
|
|
|
}
|
|
|
|
|
|
|
|
type checksumReader struct {
|
|
|
|
checksum []byte
|
|
|
|
io io.ReadCloser
|
|
|
|
hasher hash.Hash
|
|
|
|
}
|
|
|
|
|
|
|
|
func (reader *checksumReader) Read(p []byte) (int, error) {
|
|
|
|
n, err := reader.io.Read(p)
|
|
|
|
if err != nil {
|
|
|
|
if err == io.EOF {
|
|
|
|
// Check the checksum matches
|
|
|
|
actualChecksum := reader.hasher.Sum(nil)
|
|
|
|
if !bytes.Equal(reader.checksum, actualChecksum) {
|
2022-08-30 15:11:36 +00:00
|
|
|
return n, &checksumError{expected: reader.checksum, actual: actualChecksum}
|
2022-08-30 09:44:56 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return n, err
|
|
|
|
}
|
|
|
|
|
|
|
|
m, err := reader.hasher.Write(p[0:n])
|
2023-02-15 01:06:56 +00:00
|
|
|
contract.AssertNoErrorf(err, "error hashing input")
|
|
|
|
contract.Assertf(m == n, "wrote %d bytes, expected %d", m, n)
|
2022-08-30 09:44:56 +00:00
|
|
|
|
|
|
|
return n, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (reader *checksumReader) Close() error {
|
2022-08-30 15:11:36 +00:00
|
|
|
return reader.io.Close()
|
2022-08-30 09:44:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (source *checksumSource) Download(
|
|
|
|
version semver.Version, opSy string, arch string,
|
2023-03-03 16:36:39 +00:00
|
|
|
getHTTPResponse func(*http.Request) (io.ReadCloser, int64, error),
|
|
|
|
) (io.ReadCloser, int64, error) {
|
2023-08-25 15:26:25 +00:00
|
|
|
checksum, ok := source.checksum[fmt.Sprintf("%s-%s", opSy, arch)]
|
2022-08-30 09:44:56 +00:00
|
|
|
response, length, err := source.source.Download(version, opSy, arch, getHTTPResponse)
|
|
|
|
if err != nil {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
2023-08-25 15:26:25 +00:00
|
|
|
// If there's no checksum for this platform then skip validation.
|
|
|
|
if !ok {
|
|
|
|
return response, length, nil
|
|
|
|
}
|
2022-08-30 09:44:56 +00:00
|
|
|
|
|
|
|
return &checksumReader{
|
|
|
|
checksum: checksum,
|
|
|
|
hasher: sha256.New(),
|
|
|
|
io: response,
|
|
|
|
}, length, nil
|
|
|
|
}
|
|
|
|
|
2022-10-09 14:58:33 +00:00
|
|
|
// ProjectPlugin Information about a locally installed plugin specified by the project.
|
2022-08-18 14:31:10 +00:00
|
|
|
type ProjectPlugin struct {
|
|
|
|
Name string // the simple name of the plugin.
|
|
|
|
Kind PluginKind // the kind of the plugin (language, resource, etc).
|
|
|
|
Version *semver.Version // the plugin's semantic version, if present.
|
|
|
|
Path string // the path that a plugin is to be loaded from (this will always be a directory)
|
|
|
|
}
|
|
|
|
|
2022-10-09 14:58:33 +00:00
|
|
|
// Spec Return a PluginSpec object for this project plugin.
|
2022-08-26 14:51:14 +00:00
|
|
|
func (pp ProjectPlugin) Spec() PluginSpec {
|
|
|
|
return PluginSpec{
|
|
|
|
Name: pp.Name,
|
|
|
|
Kind: pp.Kind,
|
|
|
|
Version: pp.Version,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// PluginSpec provides basic specification for a plugin.
|
|
|
|
type PluginSpec struct {
|
2021-12-17 22:52:01 +00:00
|
|
|
Name string // the simple name of the plugin.
|
|
|
|
Kind PluginKind // the kind of the plugin (language, resource, etc).
|
|
|
|
Version *semver.Version // the plugin's semantic version, if present.
|
|
|
|
PluginDownloadURL string // an optional server to use when downloading this plugin.
|
|
|
|
PluginDir string // if set, will be used as the root plugin dir instead of ~/.pulumi/plugins.
|
2022-08-30 09:44:56 +00:00
|
|
|
|
|
|
|
// if set will be used to validate the plugin downloaded matches. This is keyed by "$os-$arch", e.g. "linux-x64".
|
|
|
|
Checksums map[string][]byte
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
|
|
|
|
2018-02-19 17:31:00 +00:00
|
|
|
// Dir gets the expected plugin directory for this plugin.
|
2022-08-26 14:51:14 +00:00
|
|
|
func (spec PluginSpec) Dir() string {
|
|
|
|
dir := fmt.Sprintf("%s-%s", spec.Kind, spec.Name)
|
|
|
|
if spec.Version != nil {
|
|
|
|
dir = fmt.Sprintf("%s-v%s", dir, spec.Version.String())
|
2018-02-19 17:31:00 +00:00
|
|
|
}
|
|
|
|
return dir
|
|
|
|
}
|
|
|
|
|
2022-09-21 19:32:20 +00:00
|
|
|
// File gets the expected filename for this plugin, excluding any platform specific suffixes (e.g. ".exe" on
|
|
|
|
// windows).
|
2022-08-26 14:51:14 +00:00
|
|
|
func (spec PluginSpec) File() string {
|
|
|
|
return fmt.Sprintf("pulumi-%s-%s", spec.Kind, spec.Name)
|
2018-02-06 15:20:04 +00:00
|
|
|
}
|
|
|
|
|
2018-02-19 17:31:00 +00:00
|
|
|
// DirPath returns the directory where this plugin should be installed.
|
2022-08-26 14:51:14 +00:00
|
|
|
func (spec PluginSpec) DirPath() (string, error) {
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
var err error
|
2022-08-26 14:51:14 +00:00
|
|
|
dir := spec.PluginDir
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
if dir == "" {
|
|
|
|
dir, err = GetPluginDir()
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-08-26 14:51:14 +00:00
|
|
|
return filepath.Join(dir, spec.Dir()), nil
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// LockFilePath returns the full path to the plugin's lock file used during installation
|
|
|
|
// to prevent concurrent installs.
|
2022-08-26 14:51:14 +00:00
|
|
|
func (spec PluginSpec) LockFilePath() (string, error) {
|
|
|
|
dir, err := spec.DirPath()
|
2018-02-19 17:31:00 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
2018-02-06 15:20:04 +00:00
|
|
|
}
|
2023-12-12 12:19:42 +00:00
|
|
|
return dir + ".lock", nil
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// PartialFilePath returns the full path to the plugin's partial file used during installation
|
|
|
|
// to indicate installation of the plugin hasn't completed yet.
|
2022-08-26 14:51:14 +00:00
|
|
|
func (spec PluginSpec) PartialFilePath() (string, error) {
|
|
|
|
dir, err := spec.DirPath()
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
2023-12-12 12:19:42 +00:00
|
|
|
return dir + ".partial", nil
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
|
|
|
|
2022-08-26 14:51:14 +00:00
|
|
|
func (spec PluginSpec) String() string {
|
|
|
|
var version string
|
|
|
|
if v := spec.Version; v != nil {
|
|
|
|
version = fmt.Sprintf("-%s", v)
|
|
|
|
}
|
|
|
|
return spec.Name + version
|
|
|
|
}
|
|
|
|
|
|
|
|
// PluginInfo provides basic information about a plugin. Each plugin gets installed into a system-wide
|
|
|
|
// location, by default `~/.pulumi/plugins/<kind>-<name>-<version>/`. A plugin may contain multiple files,
|
|
|
|
// however the primary loadable executable must be named `pulumi-<kind>-<name>`.
|
|
|
|
type PluginInfo struct {
|
|
|
|
Name string // the simple name of the plugin.
|
|
|
|
Path string // the path that a plugin was loaded from (this will always be a directory)
|
|
|
|
Kind PluginKind // the kind of the plugin (language, resource, etc).
|
|
|
|
Version *semver.Version // the plugin's semantic version, if present.
|
|
|
|
Size int64 // the size of the plugin, in bytes.
|
|
|
|
InstallTime time.Time // the time the plugin was installed.
|
|
|
|
LastUsedTime time.Time // the last time the plugin was used.
|
|
|
|
SchemaPath string // if set, used as the path for loading and caching the schema
|
|
|
|
SchemaTime time.Time // if set and newer than the file at SchemaPath, used to invalidate a cached schema
|
|
|
|
}
|
|
|
|
|
|
|
|
// Spec returns the PluginSpec for this PluginInfo
|
2022-10-09 14:58:33 +00:00
|
|
|
func (info *PluginInfo) Spec() PluginSpec {
|
2022-08-26 14:51:14 +00:00
|
|
|
return PluginSpec{Name: info.Name, Kind: info.Kind, Version: info.Version}
|
|
|
|
}
|
|
|
|
|
2022-10-11 16:12:29 +00:00
|
|
|
func (info PluginInfo) String() string {
|
2022-08-26 14:51:14 +00:00
|
|
|
var version string
|
|
|
|
if v := info.Version; v != nil {
|
|
|
|
version = fmt.Sprintf("-%s", v)
|
|
|
|
}
|
|
|
|
return info.Name + version
|
|
|
|
}
|
|
|
|
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
// Delete removes the plugin from the cache. It also deletes any supporting files in the cache, which includes
|
|
|
|
// any files that contain the same prefix as the plugin itself.
|
2022-10-09 14:58:33 +00:00
|
|
|
func (info *PluginInfo) Delete() error {
|
2022-08-26 14:51:14 +00:00
|
|
|
dir := info.Path
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
if err := os.RemoveAll(dir); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
// Attempt to delete any leftover .partial or .lock files.
|
|
|
|
// Don't fail the operation if we can't delete these.
|
2023-12-12 12:19:42 +00:00
|
|
|
contract.IgnoreError(os.Remove(dir + ".partial"))
|
|
|
|
contract.IgnoreError(os.Remove(dir + ".lock"))
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
return nil
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
|
|
|
|
2018-02-19 18:58:03 +00:00
|
|
|
// SetFileMetadata adds extra metadata from the given file, representing this plugin's directory.
|
2018-04-09 19:51:32 +00:00
|
|
|
func (info *PluginInfo) SetFileMetadata(path string) error {
|
2018-02-19 18:58:03 +00:00
|
|
|
// Get the file info.
|
|
|
|
file, err := os.Stat(path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Next, get the size from the directory (or, if there is none, just the file).
|
2018-04-09 19:51:32 +00:00
|
|
|
size, err := getPluginSize(path)
|
2022-06-14 06:27:11 +00:00
|
|
|
if err == nil {
|
|
|
|
info.Size = size
|
|
|
|
} else {
|
|
|
|
logging.V(6).Infof("unable to get plugin dir size for %s: %v", path, err)
|
2018-02-19 18:58:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Next get the access times from the plugin binary itself.
|
|
|
|
tinfo := times.Get(file)
|
2018-02-22 17:26:19 +00:00
|
|
|
|
2022-06-14 06:27:11 +00:00
|
|
|
if tinfo.HasChangeTime() {
|
|
|
|
info.InstallTime = tinfo.ChangeTime()
|
|
|
|
} else {
|
|
|
|
info.InstallTime = tinfo.ModTime()
|
2018-02-22 17:26:19 +00:00
|
|
|
}
|
|
|
|
|
2018-02-19 18:58:03 +00:00
|
|
|
info.LastUsedTime = tinfo.AccessTime()
|
2022-06-14 06:27:11 +00:00
|
|
|
|
|
|
|
if info.Kind == ResourcePlugin {
|
2022-11-12 02:16:53 +00:00
|
|
|
var v string
|
|
|
|
if info.Version != nil {
|
|
|
|
v = "-" + info.Version.String() + "-"
|
|
|
|
}
|
|
|
|
info.SchemaPath = filepath.Join(filepath.Dir(path), "schema-"+info.Name+v+".json")
|
2022-09-21 19:32:20 +00:00
|
|
|
info.SchemaTime = tinfo.ModTime()
|
2022-06-14 06:27:11 +00:00
|
|
|
}
|
|
|
|
|
2018-02-19 18:58:03 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-08-26 14:51:14 +00:00
|
|
|
func (spec PluginSpec) GetSource() (PluginSource, error) {
|
2022-08-30 09:44:56 +00:00
|
|
|
baseSource, err := func() (PluginSource, error) {
|
|
|
|
// The plugin has a set URL use that.
|
|
|
|
if spec.PluginDownloadURL != "" {
|
|
|
|
// Support schematised URLS if the URL has a "schema" part we recognize
|
|
|
|
url, err := url.Parse(spec.PluginDownloadURL)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2023-02-11 11:05:06 +00:00
|
|
|
switch url.Scheme {
|
|
|
|
case "github":
|
2022-08-30 09:44:56 +00:00
|
|
|
return newGithubSource(url, spec.Name, spec.Kind)
|
2023-02-11 11:05:06 +00:00
|
|
|
case "gitlab":
|
|
|
|
return newGitlabSource(url, spec.Name, spec.Kind)
|
2023-02-15 19:48:02 +00:00
|
|
|
case "http", "https":
|
|
|
|
return newHTTPSource(spec.Name, spec.Kind, url), nil
|
2023-02-11 11:05:06 +00:00
|
|
|
default:
|
2023-02-15 19:48:02 +00:00
|
|
|
return nil, fmt.Errorf("unknown plugin source scheme: %s", url.Scheme)
|
2022-08-30 09:44:56 +00:00
|
|
|
}
|
2022-06-29 19:15:01 +00:00
|
|
|
}
|
|
|
|
|
2022-08-30 09:44:56 +00:00
|
|
|
// If the plugin name matches an override, download the plugin from the override URL.
|
|
|
|
if url, ok := pluginDownloadURLOverridesParsed.get(spec.Name); ok {
|
2023-02-15 19:48:02 +00:00
|
|
|
return newHTTPSource(spec.Name, spec.Kind, urlMustParse(url)), nil
|
2022-06-29 19:15:01 +00:00
|
|
|
}
|
|
|
|
|
2022-08-30 09:44:56 +00:00
|
|
|
// Use our default fallback behaviour of github then get.pulumi.com
|
|
|
|
return newFallbackSource(spec.Name, spec.Kind), nil
|
|
|
|
}()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
|
2022-08-30 09:44:56 +00:00
|
|
|
if len(spec.Checksums) != 0 {
|
|
|
|
return newChecksumSource(baseSource, spec.Checksums), nil
|
|
|
|
}
|
|
|
|
return baseSource, nil
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// GetLatestVersion tries to find the latest version for this plugin. This is currently only supported for
|
|
|
|
// plugins we can get from github releases.
|
2022-08-26 14:51:14 +00:00
|
|
|
func (spec PluginSpec) GetLatestVersion() (*semver.Version, error) {
|
|
|
|
source, err := spec.GetSource()
|
2022-06-29 19:15:01 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-03-24 23:59:17 +00:00
|
|
|
return source.GetLatestVersion(getHTTPResponseWithRetry)
|
2022-03-03 09:22:21 +00:00
|
|
|
}
|
|
|
|
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
// Download fetches an io.ReadCloser for this plugin and also returns the size of the response (if known).
|
2022-08-26 14:51:14 +00:00
|
|
|
func (spec PluginSpec) Download() (io.ReadCloser, int64, error) {
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
// Figure out the OS/ARCH pair for the download URL.
|
2022-01-20 15:50:11 +00:00
|
|
|
var opSy string
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
switch runtime.GOOS {
|
|
|
|
case "darwin", "linux", "windows":
|
2022-01-20 15:50:11 +00:00
|
|
|
opSy = runtime.GOOS
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
default:
|
2023-01-19 20:22:54 +00:00
|
|
|
return nil, -1, fmt.Errorf("unsupported plugin OS: %s", runtime.GOOS)
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
}
|
|
|
|
var arch string
|
|
|
|
switch runtime.GOARCH {
|
2021-03-10 16:28:55 +00:00
|
|
|
case "amd64", "arm64":
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
arch = runtime.GOARCH
|
|
|
|
default:
|
2023-01-19 20:22:54 +00:00
|
|
|
return nil, -1, fmt.Errorf("unsupported plugin architecture: %s", runtime.GOARCH)
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
}
|
|
|
|
|
2021-11-29 23:21:55 +00:00
|
|
|
// The plugin version is necessary for the endpoint. If it's not present, return an error.
|
2022-08-26 14:51:14 +00:00
|
|
|
if spec.Version == nil {
|
2023-01-19 20:22:54 +00:00
|
|
|
return nil, -1, fmt.Errorf("unknown version for plugin %s", spec.Name)
|
2021-11-29 23:21:55 +00:00
|
|
|
}
|
|
|
|
|
2022-08-26 14:51:14 +00:00
|
|
|
source, err := spec.GetSource()
|
2022-06-29 19:15:01 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
2022-08-26 14:51:14 +00:00
|
|
|
return source.Download(*spec.Version, opSy, arch, getHTTPResponse)
|
2022-01-20 15:50:11 +00:00
|
|
|
}
|
2020-05-29 21:59:25 +00:00
|
|
|
|
2023-02-11 11:05:06 +00:00
|
|
|
func buildHTTPRequest(pluginEndpoint string, authorization string) (*http.Request, error) {
|
2022-01-20 15:50:11 +00:00
|
|
|
req, err := http.NewRequest("GET", pluginEndpoint, nil)
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
if err != nil {
|
2022-02-17 21:20:29 +00:00
|
|
|
return nil, err
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
userAgent := fmt.Sprintf("pulumi-cli/1 (%s; %s)", version.Version, runtime.GOOS)
|
|
|
|
req.Header.Set("User-Agent", userAgent)
|
|
|
|
|
2023-02-11 11:05:06 +00:00
|
|
|
if authorization != "" {
|
|
|
|
req.Header.Set("Authorization", authorization)
|
2022-02-17 21:20:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return req, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func getHTTPResponse(req *http.Request) (io.ReadCloser, int64, error) {
|
|
|
|
logging.V(9).Infof("full plugin download url: %s", req.URL)
|
2022-07-14 16:02:38 +00:00
|
|
|
// This logs at level 11 because it could include authentication headers, we reserve log level 11 for
|
|
|
|
// detailed api logs that may include credentials.
|
|
|
|
logging.V(11).Infof("plugin install request headers: %v", req.Header)
|
2020-10-13 12:09:29 +00:00
|
|
|
|
2023-03-24 23:59:17 +00:00
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// As above this might include authentication information, but also to be consistent at what level headers
|
|
|
|
// print at.
|
|
|
|
logging.V(11).Infof("plugin install response headers: %v", resp.Header)
|
|
|
|
|
|
|
|
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
|
|
|
contract.IgnoreClose(resp.Body)
|
|
|
|
return nil, -1, newDownloadError(resp.StatusCode, req.URL, resp.Header)
|
|
|
|
}
|
|
|
|
|
|
|
|
return resp.Body, resp.ContentLength, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func getHTTPResponseWithRetry(req *http.Request) (io.ReadCloser, int64, error) {
|
|
|
|
logging.V(9).Infof("full plugin download url: %s", req.URL)
|
|
|
|
// This logs at level 11 because it could include authentication headers, we reserve log level 11 for
|
|
|
|
// detailed api logs that may include credentials.
|
|
|
|
logging.V(11).Infof("plugin install request headers: %v", req.Header)
|
|
|
|
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
resp, err := httputil.DoWithRetry(req, http.DefaultClient)
|
|
|
|
if err != nil {
|
|
|
|
return nil, -1, err
|
|
|
|
}
|
|
|
|
|
2022-07-14 16:02:38 +00:00
|
|
|
// As above this might include authentication information, but also to be consistent at what level headers
|
|
|
|
// print at.
|
|
|
|
logging.V(11).Infof("plugin install response headers: %v", resp.Header)
|
2020-10-13 12:09:29 +00:00
|
|
|
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
2023-03-02 13:34:36 +00:00
|
|
|
contract.IgnoreClose(resp.Body)
|
|
|
|
return nil, -1, newDownloadError(resp.StatusCode, req.URL, resp.Header)
|
2022-09-20 13:58:05 +00:00
|
|
|
}
|
2022-03-16 16:00:21 +00:00
|
|
|
|
2022-09-20 13:58:05 +00:00
|
|
|
return resp.Body, resp.ContentLength, nil
|
|
|
|
}
|
2022-03-16 16:00:21 +00:00
|
|
|
|
2022-09-21 11:23:14 +00:00
|
|
|
// downloadError is an error that happened during the HTTP download of a plugin.
|
|
|
|
type downloadError struct {
|
2023-03-02 13:34:36 +00:00
|
|
|
msg string
|
|
|
|
code int
|
|
|
|
header http.Header
|
2022-09-20 13:58:05 +00:00
|
|
|
}
|
2022-03-16 16:00:21 +00:00
|
|
|
|
2022-09-21 11:23:14 +00:00
|
|
|
func (e *downloadError) Error() string {
|
2022-09-20 13:58:05 +00:00
|
|
|
return e.msg
|
|
|
|
}
|
|
|
|
|
2022-09-21 11:23:14 +00:00
|
|
|
// Create a new downloadError with a message that indicates GITHUB_TOKEN should be set.
|
2022-09-20 13:58:05 +00:00
|
|
|
func newGithubPrivateRepoError(statusCode int, url *url.URL) error {
|
2022-09-21 11:23:14 +00:00
|
|
|
return &downloadError{
|
2022-09-20 13:58:05 +00:00
|
|
|
code: statusCode,
|
|
|
|
msg: fmt.Sprintf("%d HTTP error fetching plugin from %s. "+
|
|
|
|
"If this is a private GitHub repository, try "+
|
|
|
|
"providing a token via the GITHUB_TOKEN environment variable. "+
|
|
|
|
"See: https://github.com/settings/tokens",
|
|
|
|
statusCode, url),
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
}
|
2022-09-20 13:58:05 +00:00
|
|
|
}
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
|
2022-09-21 11:23:14 +00:00
|
|
|
// Create a new downloadError.
|
2023-03-02 13:34:36 +00:00
|
|
|
func newDownloadError(statusCode int, url *url.URL, header http.Header) error {
|
2022-09-20 13:58:05 +00:00
|
|
|
if url.Host == "api.github.com" && statusCode == 404 {
|
|
|
|
return newGithubPrivateRepoError(statusCode, url)
|
|
|
|
}
|
2022-09-21 11:23:14 +00:00
|
|
|
return &downloadError{
|
2023-03-02 13:34:36 +00:00
|
|
|
code: statusCode,
|
|
|
|
msg: fmt.Sprintf("%d HTTP error fetching plugin from %s", statusCode, url),
|
|
|
|
header: header,
|
2022-09-20 13:58:05 +00:00
|
|
|
}
|
Add `--server` to `pulumi plugin install`
Previously, when the CLI wanted to install a plugin, it used a special
method, `DownloadPlugin` on the `httpstate` backend to actually fetch
the tarball that had the plugin. The reason for this is largely tied
to history, at one point during a closed beta, we required presenting
an API key to download plugins (as a way to enforce folks outside the
beta could not download them) and because of that it was natural to
bake that functionality into the part of the code that interfaced with
the rest of the API from the Pulumi Service.
The downside here is that it means we need to host all the plugins on
`api.pulumi.com` which prevents community folks from being able to
easily write resource providers, since they have to manually manage
the process of downloading a provider to a machine and getting it on
the `$PATH` or putting it in the plugin cache.
To make this easier, we add a `--server` argument you can pass to
`pulumi plugin install` to control the URL that it attempts to fetch
the tarball from. We still have perscriptive guidence on how the
tarball must be
named (`pulumi-[<type>]-[<provider-name>]-vX.Y.Z.tar.gz`) but the base
URL can now be configured.
Folks publishing packages can use install scripts to run `pulumi
plugin install` passing a custom `--server` argument, if needed.
There are two improvements we can make to provide a nicer end to end
story here:
- We can augment the GetRequiredPlugins method on the language
provider to also return information about an optional server to use
when downloading the provider.
- We can pass information about a server to download plugins from as
part of a resource registration or creation of a first class
provider.
These help out in cases where for one reason or another where `pulumi
plugin install` doesn't get run before an update takes place and would
allow us to either do the right thing ahead of time or provide better
error messages with the correct `--server` argument. But, for now,
this unblocks a majority of the cases we care about and provides a
path forward for folks that want to develop and host their own
resource providers.
2019-05-30 20:56:55 +00:00
|
|
|
}
|
|
|
|
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
// installLock acquires a file lock used to prevent concurrent installs.
|
2022-08-26 14:51:14 +00:00
|
|
|
func (spec PluginSpec) installLock() (unlock func(), err error) {
|
|
|
|
finalDir, err := spec.DirPath()
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-12-12 12:19:42 +00:00
|
|
|
lockFilePath := finalDir + ".lock"
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
|
2023-03-03 16:36:39 +00:00
|
|
|
if err := os.MkdirAll(filepath.Dir(lockFilePath), 0o700); err != nil {
|
2023-01-19 20:22:54 +00:00
|
|
|
return nil, fmt.Errorf("creating plugin root: %w", err)
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
mutex := fsutil.NewFileMutex(lockFilePath)
|
|
|
|
if err := mutex.Lock(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return func() {
|
|
|
|
contract.IgnoreError(mutex.Unlock())
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2022-06-09 21:57:56 +00:00
|
|
|
// Install installs a plugin's tarball into the cache. See InstallWithContext for details.
|
2022-08-26 14:51:14 +00:00
|
|
|
func (spec PluginSpec) Install(tgz io.ReadCloser, reinstall bool) error {
|
|
|
|
return spec.InstallWithContext(context.Background(), tarPlugin{tgz}, reinstall)
|
2022-07-15 04:32:40 +00:00
|
|
|
}
|
|
|
|
|
2023-03-20 20:13:33 +00:00
|
|
|
// pluginDownloader is responsible for downloading plugins from PluginSpecs.
|
|
|
|
//
|
|
|
|
// It allows hooking into various stages of the download process
|
|
|
|
// to allow for custom behavior and progress reporting.
|
|
|
|
//
|
|
|
|
// All fields are optional.
|
|
|
|
type pluginDownloader struct {
|
|
|
|
// WrapStream wraps the stream returned by the plugin source.
|
|
|
|
// This is useful for things like reporting progress.
|
|
|
|
WrapStream func(stream io.ReadCloser, size int64) io.ReadCloser
|
|
|
|
|
|
|
|
// OnRetry receives a notification when a download fails
|
|
|
|
// and is about to be retried.
|
|
|
|
// err is the error that caused the retry.
|
|
|
|
// attempt is the number of the attempt that failed (starting at 1).
|
|
|
|
// limit is the maximum number of attempts.
|
|
|
|
// delay is the amount of time that will be slept before the next attempt.
|
|
|
|
// DO NOT sleep in this function. It's for observation only.
|
|
|
|
OnRetry func(err error, attempt int, limit int, delay time.Duration)
|
2023-03-20 20:20:38 +00:00
|
|
|
|
|
|
|
// Controls how to sleep between retries.
|
|
|
|
After func(time.Duration) <-chan time.Time // == time.After
|
2023-03-20 20:13:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// copyBuffer copies from src to dst until either EOF is reached on src or an error occurs.
|
|
|
|
//
|
|
|
|
// This is an internal helper that's pretty much just a copy of io.Copy except it returns read and
|
|
|
|
// write errors separately. We only want to retry if the read (i.e. download) fails, if the write
|
|
|
|
// fails thats probably due to file permissions or space limitations and there's no point retrying.
|
|
|
|
func (d *pluginDownloader) copyBuffer(dst io.Writer, src io.Reader) (written int64, readErr error, writeErr error) {
|
|
|
|
size := 32 * 1024
|
|
|
|
if l, ok := src.(*io.LimitedReader); ok && int64(size) > l.N {
|
|
|
|
if l.N < 1 {
|
|
|
|
size = 1
|
|
|
|
} else {
|
|
|
|
size = int(l.N)
|
2022-08-10 15:59:27 +00:00
|
|
|
}
|
2023-03-20 20:13:33 +00:00
|
|
|
}
|
|
|
|
buf := make([]byte, size)
|
|
|
|
|
|
|
|
for {
|
|
|
|
nr, er := src.Read(buf)
|
|
|
|
if nr > 0 {
|
|
|
|
nw, ew := dst.Write(buf[0:nr])
|
|
|
|
if nw < 0 || nr < nw {
|
|
|
|
nw = 0
|
|
|
|
if ew == nil {
|
|
|
|
ew = errors.New("invalid write result")
|
2022-08-10 15:59:27 +00:00
|
|
|
}
|
|
|
|
}
|
2023-03-20 20:13:33 +00:00
|
|
|
written += int64(nw)
|
|
|
|
if ew != nil {
|
|
|
|
return written, nil, ew
|
|
|
|
}
|
|
|
|
if nr != nw {
|
|
|
|
return written, nil, io.ErrShortWrite
|
2022-08-10 15:59:27 +00:00
|
|
|
}
|
|
|
|
}
|
2023-03-20 20:13:33 +00:00
|
|
|
if er != nil {
|
|
|
|
if er == io.EOF {
|
|
|
|
er = nil
|
|
|
|
}
|
|
|
|
return written, er, nil
|
2022-07-27 10:13:17 +00:00
|
|
|
}
|
|
|
|
}
|
2023-03-20 20:13:33 +00:00
|
|
|
}
|
2022-07-27 10:13:17 +00:00
|
|
|
|
2023-03-20 20:13:33 +00:00
|
|
|
func (d *pluginDownloader) tryDownload(pkgPlugin PluginSpec, dst io.WriteCloser) (error, error) {
|
|
|
|
defer dst.Close()
|
|
|
|
tarball, expectedByteCount, err := pkgPlugin.Download()
|
|
|
|
if err != nil {
|
|
|
|
return err, nil
|
|
|
|
}
|
|
|
|
if d.WrapStream != nil {
|
|
|
|
tarball = d.WrapStream(tarball, expectedByteCount)
|
|
|
|
}
|
|
|
|
defer tarball.Close()
|
|
|
|
copiedByteCount, readErr, writerErr := d.copyBuffer(dst, tarball)
|
|
|
|
if readErr != nil || writerErr != nil {
|
|
|
|
return readErr, writerErr
|
|
|
|
}
|
|
|
|
if copiedByteCount != expectedByteCount {
|
|
|
|
return nil, fmt.Errorf("expected %d bytes but copied %d when downloading plugin %s",
|
|
|
|
expectedByteCount, copiedByteCount, pkgPlugin)
|
|
|
|
}
|
|
|
|
return nil, nil
|
|
|
|
}
|
2022-08-10 15:59:27 +00:00
|
|
|
|
2023-03-20 20:13:33 +00:00
|
|
|
func (d *pluginDownloader) tryDownloadToFile(pkgPlugin PluginSpec) (string, error, error) {
|
|
|
|
file, err := os.CreateTemp("" /* default temp dir */, "pulumi-plugin-tar")
|
|
|
|
if err != nil {
|
|
|
|
return "", nil, err
|
|
|
|
}
|
|
|
|
readErr, writeErr := d.tryDownload(pkgPlugin, file)
|
2023-06-09 14:35:58 +00:00
|
|
|
logging.V(10).Infof("try downloaded plugin %s to %s: %v %v", pkgPlugin, file.Name(), readErr, writeErr)
|
2023-03-20 20:13:33 +00:00
|
|
|
if readErr != nil || writeErr != nil {
|
|
|
|
err2 := os.Remove(file.Name())
|
|
|
|
if err2 != nil {
|
|
|
|
// only one of readErr or writeErr will be set
|
|
|
|
err := readErr
|
|
|
|
if err == nil {
|
|
|
|
err = writeErr
|
2022-07-27 10:13:17 +00:00
|
|
|
}
|
2023-03-20 20:13:33 +00:00
|
|
|
|
|
|
|
return "", nil, fmt.Errorf("error while removing tempfile: %v. Context: %w", err2, err)
|
2022-07-27 10:13:17 +00:00
|
|
|
}
|
2023-03-20 20:13:33 +00:00
|
|
|
return "", readErr, writeErr
|
2022-07-27 10:13:17 +00:00
|
|
|
}
|
2023-03-20 20:13:33 +00:00
|
|
|
return file.Name(), nil, nil
|
|
|
|
}
|
2022-07-27 10:13:17 +00:00
|
|
|
|
2023-03-20 20:13:33 +00:00
|
|
|
func (d *pluginDownloader) downloadToFileWithRetry(pkgPlugin PluginSpec) (string, error) {
|
|
|
|
delay := 80 * time.Millisecond
|
2023-03-20 20:20:38 +00:00
|
|
|
backoff := 2.0
|
|
|
|
maxAttempts := 5
|
|
|
|
|
|
|
|
_, path, err := (&retry.Retryer{
|
|
|
|
After: d.After,
|
|
|
|
}).Until(context.Background(), retry.Acceptor{
|
|
|
|
Delay: &delay,
|
|
|
|
Backoff: &backoff,
|
|
|
|
Accept: func(attempt int, nextRetryTime time.Duration) (bool, interface{}, error) {
|
|
|
|
if attempt >= maxAttempts {
|
|
|
|
return false, nil, fmt.Errorf("failed all %d attempts", maxAttempts)
|
|
|
|
}
|
2022-07-27 10:13:17 +00:00
|
|
|
|
2023-03-20 20:20:38 +00:00
|
|
|
tempFile, readErr, writeErr := d.tryDownloadToFile(pkgPlugin)
|
|
|
|
if readErr == nil && writeErr == nil {
|
|
|
|
return true, tempFile, nil
|
|
|
|
}
|
|
|
|
if writeErr != nil {
|
|
|
|
// Writes are local. If they fail,
|
|
|
|
// there's no point retrying.
|
|
|
|
return false, "", writeErr
|
|
|
|
}
|
2022-09-20 13:58:05 +00:00
|
|
|
|
2023-03-20 20:20:38 +00:00
|
|
|
// If the readErr is a checksum error don't retry.
|
|
|
|
var checksumErr *checksumError
|
|
|
|
if errors.As(readErr, &checksumErr) {
|
|
|
|
return false, "", readErr
|
|
|
|
}
|
2022-09-20 13:58:05 +00:00
|
|
|
|
2023-03-20 20:20:38 +00:00
|
|
|
// Don't retry, since the request was processed and rejected.
|
|
|
|
var downloadErr *downloadError
|
|
|
|
if errors.As(readErr, &downloadErr) && (downloadErr.code == 404 || downloadErr.code == 403) {
|
|
|
|
return false, "", readErr
|
|
|
|
}
|
|
|
|
|
|
|
|
if d.OnRetry != nil {
|
|
|
|
d.OnRetry(readErr, attempt+1, maxAttempts, nextRetryTime)
|
|
|
|
}
|
|
|
|
|
|
|
|
return false, "", nil
|
|
|
|
},
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
2022-07-27 10:13:17 +00:00
|
|
|
}
|
2023-03-20 20:20:38 +00:00
|
|
|
|
|
|
|
return path.(string), nil
|
2023-03-20 20:13:33 +00:00
|
|
|
}
|
2022-07-27 10:13:17 +00:00
|
|
|
|
2023-03-20 20:13:33 +00:00
|
|
|
// DownloadToFile downloads the given PluginSpec to a temporary file
|
|
|
|
// and returns that temporary file.
|
|
|
|
//
|
|
|
|
// This has some retry logic to re-attempt the download if it errors for any reason.
|
|
|
|
func (d *pluginDownloader) DownloadToFile(pkgPlugin PluginSpec) (*os.File, error) {
|
|
|
|
tarball, err := d.downloadToFileWithRetry(pkgPlugin)
|
2022-07-27 10:13:17 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("failed to download plugin: %s: %w", pkgPlugin, err)
|
|
|
|
}
|
|
|
|
reader, err := os.Open(tarball)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("failed to open downloaded plugin: %s: %w", pkgPlugin, err)
|
|
|
|
}
|
|
|
|
return reader, nil
|
|
|
|
}
|
|
|
|
|
2023-03-20 20:13:33 +00:00
|
|
|
// DownloadToFile downloads the given PluginInfo to a temporary file and returns that temporary file.
|
|
|
|
// This has some retry logic to re-attempt the download if it errors for any reason.
|
|
|
|
func DownloadToFile(
|
|
|
|
pkgPlugin PluginSpec,
|
|
|
|
wrapper func(stream io.ReadCloser, size int64) io.ReadCloser,
|
|
|
|
retry func(err error, attempt int, limit int, delay time.Duration),
|
|
|
|
) (*os.File, error) {
|
|
|
|
return (&pluginDownloader{
|
|
|
|
WrapStream: wrapper,
|
|
|
|
OnRetry: retry,
|
|
|
|
}).DownloadToFile(pkgPlugin)
|
|
|
|
}
|
|
|
|
|
2022-07-15 04:32:40 +00:00
|
|
|
type PluginContent interface {
|
|
|
|
io.Closer
|
|
|
|
|
|
|
|
writeToDir(pathToDir string) error
|
|
|
|
}
|
|
|
|
|
2022-08-26 14:51:14 +00:00
|
|
|
func SingleFilePlugin(f *os.File, spec PluginSpec) PluginContent {
|
|
|
|
return singleFilePlugin{F: f, Kind: spec.Kind, Name: spec.Name}
|
2022-07-15 04:32:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type singleFilePlugin struct {
|
|
|
|
F *os.File
|
|
|
|
Kind PluginKind
|
|
|
|
Name string
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p singleFilePlugin) writeToDir(finalDir string) error {
|
2023-01-06 22:39:16 +00:00
|
|
|
bytes, err := io.ReadAll(p.F)
|
2022-07-15 04:32:40 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
finalPath := filepath.Join(finalDir, fmt.Sprintf("pulumi-%s-%s", p.Kind, p.Name))
|
|
|
|
// We are writing an executable.
|
2023-03-03 16:36:39 +00:00
|
|
|
return os.WriteFile(finalPath, bytes, 0o700) //nolint:gosec
|
2022-07-15 04:32:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (p singleFilePlugin) Close() error {
|
|
|
|
return p.F.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
func TarPlugin(tgz io.ReadCloser) PluginContent {
|
|
|
|
return tarPlugin{Tgz: tgz}
|
|
|
|
}
|
|
|
|
|
|
|
|
type tarPlugin struct {
|
|
|
|
Tgz io.ReadCloser
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p tarPlugin) Close() error {
|
|
|
|
return p.Tgz.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p tarPlugin) writeToDir(finalPath string) error {
|
|
|
|
return archive.ExtractTGZ(p.Tgz, finalPath)
|
|
|
|
}
|
|
|
|
|
|
|
|
func DirPlugin(rootPath string) PluginContent {
|
|
|
|
return dirPlugin{Root: rootPath}
|
|
|
|
}
|
|
|
|
|
|
|
|
type dirPlugin struct {
|
|
|
|
Root string
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p dirPlugin) Close() error {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p dirPlugin) writeToDir(dstRoot string) error {
|
|
|
|
return filepath.WalkDir(p.Root, func(srcPath string, d fs.DirEntry, err error) error {
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
relPath := strings.TrimPrefix(srcPath, p.Root)
|
|
|
|
dstPath := filepath.Join(dstRoot, relPath)
|
|
|
|
|
|
|
|
if srcPath == p.Root {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
if d.IsDir() {
|
2023-03-03 16:36:39 +00:00
|
|
|
return os.Mkdir(dstPath, 0o700)
|
2022-07-15 04:32:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
src, err := os.Open(srcPath)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
info, err := d.Info()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-01-06 22:39:16 +00:00
|
|
|
bytes, err := io.ReadAll(src)
|
2022-07-15 04:32:40 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return os.WriteFile(dstPath, bytes, info.Mode())
|
|
|
|
})
|
2022-06-09 21:57:56 +00:00
|
|
|
}
|
|
|
|
|
2022-10-11 16:12:29 +00:00
|
|
|
// InstallWithContext installs a plugin's tarball into the cache. It validates that plugin names are in the expected
|
|
|
|
// format. Previous versions of Pulumi extracted the tarball to a temp directory first, and then renamed the temp
|
|
|
|
// directory to the final directory. The rename operation fails often enough on Windows due to aggressive virus scanners
|
|
|
|
// opening files in the temp directory. To address this, we now extract the tarball directly into the final directory,
|
|
|
|
// and use file locks to prevent concurrent installs.
|
|
|
|
//
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
// Each plugin has its own file lock, with the same name as the plugin directory, with a `.lock` suffix.
|
|
|
|
// During installation an empty file with a `.partial` suffix is created, indicating that installation is in-progress.
|
|
|
|
// The `.partial` file is deleted when installation is complete, indicating that the plugin has finished installing.
|
|
|
|
// If a failure occurs during installation, the `.partial` file will remain, indicating the plugin wasn't fully
|
|
|
|
// installed. The next time the plugin is installed, the old installation directory will be removed and replaced with
|
2022-10-09 14:58:33 +00:00
|
|
|
// a fresh installation.
|
2022-08-26 14:51:14 +00:00
|
|
|
func (spec PluginSpec) InstallWithContext(ctx context.Context, content PluginContent, reinstall bool) error {
|
2022-07-15 04:32:40 +00:00
|
|
|
defer contract.IgnoreClose(content)
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
|
|
|
|
// Fetch the directory into which we will expand this tarball.
|
2022-08-26 14:51:14 +00:00
|
|
|
finalDir, err := spec.DirPath()
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-11-09 22:59:13 +00:00
|
|
|
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
// Create a file lock file at <pluginsdir>/<kind>-<name>-<version>.lock.
|
2022-08-26 14:51:14 +00:00
|
|
|
unlock, err := spec.installLock()
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer unlock()
|
|
|
|
|
|
|
|
// Cleanup any temp dirs from failed installations of this plugin from previous versions of Pulumi.
|
|
|
|
if err := cleanupTempDirs(finalDir); err != nil {
|
|
|
|
// We don't want to fail the installation if there was an error cleaning up these old temp dirs.
|
|
|
|
// Instead, log the error and continue on.
|
2023-12-20 15:54:06 +00:00
|
|
|
logging.V(5).Infof("Install: Error cleaning up temp dirs: %s", err)
|
2018-11-17 00:43:18 +00:00
|
|
|
}
|
|
|
|
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
// Get the partial file path (e.g. <pluginsdir>/<kind>-<name>-<version>.partial).
|
2022-08-26 14:51:14 +00:00
|
|
|
partialFilePath, err := spec.PartialFilePath()
|
2018-11-15 19:54:45 +00:00
|
|
|
if err != nil {
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
return err
|
2018-02-19 17:31:00 +00:00
|
|
|
}
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
// Check whether the directory exists while we were waiting on the lock.
|
|
|
|
_, finalDirStatErr := os.Stat(finalDir)
|
|
|
|
if finalDirStatErr == nil {
|
|
|
|
_, partialFileStatErr := os.Stat(partialFilePath)
|
|
|
|
if partialFileStatErr != nil {
|
2022-02-03 07:10:16 +00:00
|
|
|
if !os.IsNotExist(partialFileStatErr) {
|
|
|
|
return partialFileStatErr
|
|
|
|
}
|
|
|
|
if !reinstall {
|
|
|
|
// finalDir exists, there's no partial file, and we're not reinstalling, so the plugin is already
|
|
|
|
// installed.
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
2018-11-15 19:54:45 +00:00
|
|
|
|
2022-02-03 07:10:16 +00:00
|
|
|
// Either the partial file exists--meaning a previous attempt at installing the plugin failed--or we're
|
|
|
|
// deliberately reinstalling the plugin. Delete finalDir so we can try installing again. There's no need to
|
|
|
|
// delete the partial file since we'd just be recreating it again below anyway.
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
if err := os.RemoveAll(finalDir); err != nil {
|
2019-08-01 18:05:57 +00:00
|
|
|
return err
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
} else if !os.IsNotExist(finalDirStatErr) {
|
|
|
|
return finalDirStatErr
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create an empty partial file to indicate installation is in-progress.
|
2023-03-03 16:36:39 +00:00
|
|
|
if err := os.WriteFile(partialFilePath, nil, 0o600); err != nil {
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
return err
|
|
|
|
}
|
2018-11-17 01:53:57 +00:00
|
|
|
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
// Create the final directory.
|
2023-03-03 16:36:39 +00:00
|
|
|
if err := os.MkdirAll(finalDir, 0o700); err != nil {
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2022-07-15 04:32:40 +00:00
|
|
|
if err := content.writeToDir(finalDir); err != nil {
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
return err
|
|
|
|
}
|
2018-11-09 22:59:13 +00:00
|
|
|
|
2020-11-21 16:02:05 +00:00
|
|
|
// Even though we deferred closing the tarball at the beginning of this function, go ahead and explicitly close
|
|
|
|
// it now since we're finished extracting it, to prevent subsequent output from being displayed oddly with
|
|
|
|
// the progress bar.
|
2022-07-15 04:32:40 +00:00
|
|
|
contract.IgnoreClose(content)
|
2020-11-21 16:02:05 +00:00
|
|
|
|
2020-09-14 20:54:26 +00:00
|
|
|
// Install dependencies, if needed.
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
proj, err := LoadPluginProject(filepath.Join(finalDir, "PulumiPlugin.yaml"))
|
2020-09-14 20:54:26 +00:00
|
|
|
if err != nil && !os.IsNotExist(err) {
|
2023-01-19 20:22:54 +00:00
|
|
|
return fmt.Errorf("loading PulumiPlugin.yaml: %w", err)
|
2020-09-14 20:54:26 +00:00
|
|
|
}
|
|
|
|
if proj != nil {
|
|
|
|
runtime := strings.ToLower(proj.Runtime.Name())
|
|
|
|
// For now, we only do this for Node.js and Python. For Go, the expectation is the binary is
|
|
|
|
// already built. For .NET, similarly, a single self-contained binary could be used, but
|
|
|
|
// otherwise `dotnet run` will implicitly run `dotnet restore`.
|
|
|
|
// TODO[pulumi/pulumi#1334]: move to the language plugins so we don't have to hard code here.
|
|
|
|
switch runtime {
|
|
|
|
case "nodejs":
|
2020-11-21 16:02:05 +00:00
|
|
|
var b bytes.Buffer
|
2022-06-09 21:57:56 +00:00
|
|
|
if _, err := npm.Install(ctx, finalDir, true /* production */, &b, &b); err != nil {
|
2020-11-21 16:02:05 +00:00
|
|
|
os.Stderr.Write(b.Bytes())
|
2023-01-19 20:22:54 +00:00
|
|
|
return fmt.Errorf("installing plugin dependencies: %w", err)
|
2020-09-14 20:54:26 +00:00
|
|
|
}
|
|
|
|
case "python":
|
2022-06-09 21:57:56 +00:00
|
|
|
if err := python.InstallDependencies(ctx, finalDir, "venv", false /*showOutput*/); err != nil {
|
2023-01-19 20:22:54 +00:00
|
|
|
return fmt.Errorf("installing plugin dependencies: %w", err)
|
2020-09-14 20:54:26 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
// Installation is complete. Remove the partial file.
|
|
|
|
return os.Remove(partialFilePath)
|
|
|
|
}
|
|
|
|
|
|
|
|
// cleanupTempDirs cleans up leftover temp dirs from failed installs with previous versions of Pulumi.
|
|
|
|
func cleanupTempDirs(finalDir string) error {
|
|
|
|
dir := filepath.Dir(finalDir)
|
|
|
|
|
2022-09-10 07:22:11 +00:00
|
|
|
infos, err := os.ReadDir(dir)
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, info := range infos {
|
|
|
|
// Temp dirs have a suffix of `.tmpXXXXXX` (where `XXXXXX`) is a random number,
|
2023-01-06 22:39:16 +00:00
|
|
|
// from os.CreateTemp.
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
if info.IsDir() && installingPluginRegexp.MatchString(info.Name()) {
|
|
|
|
path := filepath.Join(dir, info.Name())
|
|
|
|
if err := os.RemoveAll(path); err != nil {
|
2023-01-19 20:22:54 +00:00
|
|
|
return fmt.Errorf("cleaning up temp dir %s: %w", path, err)
|
2019-08-01 00:44:26 +00:00
|
|
|
}
|
|
|
|
}
|
2018-11-16 00:25:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// PluginKind represents a kind of a plugin that may be dynamically loaded and used by Pulumi.
|
|
|
|
type PluginKind string
|
|
|
|
|
|
|
|
const (
|
|
|
|
// AnalyzerPlugin is a plugin that can be used as a resource analyzer.
|
|
|
|
AnalyzerPlugin PluginKind = "analyzer"
|
|
|
|
// LanguagePlugin is a plugin that can be used as a language host.
|
|
|
|
LanguagePlugin PluginKind = "language"
|
|
|
|
// ResourcePlugin is a plugin that can be used as a resource provider for custom CRUD operations.
|
|
|
|
ResourcePlugin PluginKind = "resource"
|
2023-03-04 09:13:30 +00:00
|
|
|
// ConverterPlugin is a plugin that can be used to convert from other ecosystems to Pulumi.
|
|
|
|
ConverterPlugin PluginKind = "converter"
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// IsPluginKind returns true if k is a valid plugin kind, and false otherwise.
|
|
|
|
func IsPluginKind(k string) bool {
|
|
|
|
switch PluginKind(k) {
|
2023-03-04 09:13:30 +00:00
|
|
|
case AnalyzerPlugin, LanguagePlugin, ResourcePlugin, ConverterPlugin:
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
return true
|
|
|
|
default:
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-17 19:42:27 +00:00
|
|
|
// HasPlugin returns true if the given plugin exists.
|
2022-08-26 14:51:14 +00:00
|
|
|
func HasPlugin(spec PluginSpec) bool {
|
|
|
|
dir, err := spec.DirPath()
|
2018-02-17 19:42:27 +00:00
|
|
|
if err == nil {
|
2018-02-19 17:31:00 +00:00
|
|
|
_, err := os.Stat(dir)
|
2018-02-17 19:42:27 +00:00
|
|
|
if err == nil {
|
2022-08-26 14:51:14 +00:00
|
|
|
partialFilePath, err := spec.PartialFilePath()
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
if err == nil {
|
|
|
|
if _, err := os.Stat(partialFilePath); os.IsNotExist(err) {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
2018-02-17 19:42:27 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2018-04-05 23:37:50 +00:00
|
|
|
// HasPluginGTE returns true if the given plugin exists at the given version number or greater.
|
2022-08-26 14:51:14 +00:00
|
|
|
func HasPluginGTE(spec PluginSpec) (bool, error) {
|
2018-04-05 23:37:50 +00:00
|
|
|
// If an exact match, return true right away.
|
2022-08-26 14:51:14 +00:00
|
|
|
if HasPlugin(spec) {
|
2018-04-05 23:37:50 +00:00
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Otherwise, load up the list of plugins and find one with the same name/type and >= version.
|
|
|
|
plugs, err := GetPlugins()
|
|
|
|
if err != nil {
|
|
|
|
return false, err
|
|
|
|
}
|
2019-03-01 23:42:38 +00:00
|
|
|
|
|
|
|
// If we're not doing the legacy plugin behavior and we've been asked for a specific version, do the same plugin
|
|
|
|
// search that we'd do at runtime. This ensures that `pulumi plugin install` works the same way that the runtime
|
|
|
|
// loader does, to minimize confusion when a user has to install new plugins.
|
2023-12-03 09:15:07 +00:00
|
|
|
var match *PluginInfo
|
2022-08-26 14:51:14 +00:00
|
|
|
if !enableLegacyPluginBehavior && spec.Version != nil {
|
|
|
|
requestedVersion := semver.MustParseRange(spec.Version.String())
|
2023-12-03 09:15:07 +00:00
|
|
|
match = SelectCompatiblePlugin(plugs, spec.Kind, spec.Name, requestedVersion)
|
|
|
|
} else {
|
|
|
|
match = LegacySelectCompatiblePlugin(plugs, spec.Kind, spec.Name, spec.Version)
|
2018-04-05 23:37:50 +00:00
|
|
|
}
|
2023-12-03 09:15:07 +00:00
|
|
|
return match != nil, nil
|
2018-04-05 23:37:50 +00:00
|
|
|
}
|
|
|
|
|
2020-02-26 17:45:39 +00:00
|
|
|
// GetPolicyDir returns the directory in which an organization's Policy Packs on the current machine are managed.
|
2020-02-25 17:38:58 +00:00
|
|
|
func GetPolicyDir(orgName string) (string, error) {
|
|
|
|
return GetPulumiPath(PolicyDir, orgName)
|
2019-06-30 23:34:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// GetPolicyPath finds a PolicyPack by its name version, as well as a bool marked true if the path
|
|
|
|
// already exists and is a directory.
|
2020-02-25 17:38:58 +00:00
|
|
|
func GetPolicyPath(orgName, name, version string) (string, bool, error) {
|
|
|
|
policiesDir, err := GetPolicyDir(orgName)
|
2019-06-30 23:34:39 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", false, err
|
|
|
|
}
|
|
|
|
|
2019-08-11 05:17:08 +00:00
|
|
|
policyPackPath := path.Join(policiesDir, fmt.Sprintf("pulumi-analyzer-%s-v%s", name, version))
|
2019-06-30 23:34:39 +00:00
|
|
|
|
|
|
|
file, err := os.Stat(policyPackPath)
|
|
|
|
if err == nil && file.IsDir() {
|
|
|
|
// PolicyPack exists. Return.
|
|
|
|
return policyPackPath, true, nil
|
|
|
|
} else if err != nil && !os.IsNotExist(err) {
|
|
|
|
// Error trying to inspect PolicyPack FS entry. Return error.
|
|
|
|
return "", false, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Not found. Return empty path.
|
|
|
|
return policyPackPath, false, nil
|
|
|
|
}
|
|
|
|
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
// GetPluginDir returns the directory in which plugins on the current machine are managed.
|
|
|
|
func GetPluginDir() (string, error) {
|
2019-10-08 22:01:46 +00:00
|
|
|
return GetPulumiPath(PluginDir)
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
|
|
|
|
2021-05-28 14:26:08 +00:00
|
|
|
// GetPlugins returns a list of installed plugins without size info and last accessed metadata.
|
|
|
|
// Plugin size requires recursively traversing the plugin directory, which can be extremely
|
|
|
|
// expensive with the introduction of nodejs multilang components that have
|
|
|
|
// deeply nested node_modules folders.
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
func GetPlugins() ([]PluginInfo, error) {
|
|
|
|
// To get the list of plugins, simply scan the directory in the usual place.
|
|
|
|
dir, err := GetPluginDir()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2021-05-28 14:26:08 +00:00
|
|
|
return getPlugins(dir, true /* skipMetadata */)
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
}
|
|
|
|
|
2021-05-28 14:26:08 +00:00
|
|
|
// GetPluginsWithMetadata returns a list of installed plugins with metadata about size,
|
|
|
|
// and last access (POOR RUNTIME PERF). Plugin size requires recursively traversing the
|
|
|
|
// plugin directory, which can be extremely expensive with the introduction of
|
|
|
|
// nodejs multilang components that have deeply nested node_modules folders.
|
|
|
|
func GetPluginsWithMetadata() ([]PluginInfo, error) {
|
|
|
|
// To get the list of plugins, simply scan the directory in the usual place.
|
|
|
|
dir, err := GetPluginDir()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return getPlugins(dir, false /* skipMetadata */)
|
|
|
|
}
|
|
|
|
|
|
|
|
func getPlugins(dir string, skipMetadata bool) ([]PluginInfo, error) {
|
2022-09-10 07:22:11 +00:00
|
|
|
files, err := os.ReadDir(dir)
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Now read the file infos and create the plugin infos.
|
|
|
|
var plugins []PluginInfo
|
|
|
|
for _, file := range files {
|
|
|
|
// Skip anything that doesn't look like a plugin.
|
2018-02-19 17:31:00 +00:00
|
|
|
if kind, name, version, ok := tryPlugin(file); ok {
|
2022-06-14 06:27:11 +00:00
|
|
|
path := filepath.Join(dir, file.Name())
|
2018-02-19 18:58:03 +00:00
|
|
|
plugin := PluginInfo{
|
|
|
|
Name: name,
|
|
|
|
Kind: kind,
|
|
|
|
Version: &version,
|
2022-06-14 06:27:11 +00:00
|
|
|
Path: path,
|
2018-02-19 18:58:03 +00:00
|
|
|
}
|
2023-12-12 12:19:42 +00:00
|
|
|
if _, err := os.Stat(path + ".partial"); err == nil {
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
// Skip it if the partial file exists, meaning the plugin is not fully installed.
|
|
|
|
continue
|
|
|
|
} else if !os.IsNotExist(err) {
|
|
|
|
return nil, err
|
|
|
|
}
|
2021-05-28 14:26:08 +00:00
|
|
|
// computing plugin sizes can be very expensive (nested node_modules)
|
|
|
|
if !skipMetadata {
|
|
|
|
if err = plugin.SetFileMetadata(path); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2018-02-19 17:31:00 +00:00
|
|
|
}
|
2018-02-19 18:58:03 +00:00
|
|
|
plugins = append(plugins, plugin)
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return plugins, nil
|
|
|
|
}
|
|
|
|
|
2023-03-31 18:33:55 +00:00
|
|
|
// We currently bundle some plugins with "pulumi" and thus expect them to be next to the pulumi binary.
|
|
|
|
// Eventually we want to fix this so new plugins are true plugins in the plugin cache.
|
|
|
|
func IsPluginBundled(kind PluginKind, name string) bool {
|
|
|
|
return (kind == LanguagePlugin && name == "nodejs") ||
|
|
|
|
(kind == LanguagePlugin && name == "go") ||
|
|
|
|
(kind == LanguagePlugin && name == "python") ||
|
|
|
|
(kind == LanguagePlugin && name == "dotnet") ||
|
|
|
|
(kind == LanguagePlugin && name == "yaml") ||
|
|
|
|
(kind == LanguagePlugin && name == "java") ||
|
|
|
|
(kind == ResourcePlugin && name == "pulumi-nodejs") ||
|
|
|
|
(kind == ResourcePlugin && name == "pulumi-python") ||
|
|
|
|
(kind == AnalyzerPlugin && name == "policy") ||
|
|
|
|
(kind == AnalyzerPlugin && name == "policy-python")
|
|
|
|
}
|
|
|
|
|
2018-04-05 23:37:50 +00:00
|
|
|
// GetPluginPath finds a plugin's path by its kind, name, and optional version. It will match the latest version that
|
|
|
|
// is >= the version specified. If no version is supplied, the latest plugin for that given kind/name pair is loaded,
|
2021-05-04 15:30:59 +00:00
|
|
|
// using standard semver sorting rules. A plugin may be overridden entirely by placing it on your $PATH, though it is
|
|
|
|
// possible to opt out of this behavior by setting PULUMI_IGNORE_AMBIENT_PLUGINS to any non-empty value.
|
2023-08-07 12:15:57 +00:00
|
|
|
func GetPluginPath(d diag.Sink, kind PluginKind, name string, version *semver.Version,
|
2023-03-03 16:36:39 +00:00
|
|
|
projectPlugins []ProjectPlugin,
|
|
|
|
) (string, error) {
|
2023-08-07 12:15:57 +00:00
|
|
|
info, path, err := getPluginInfoAndPath(d, kind, name, version, true /* skipMetadata */, projectPlugins)
|
2022-06-14 06:27:11 +00:00
|
|
|
if err != nil {
|
2022-07-22 09:33:59 +00:00
|
|
|
return "", err
|
2022-06-14 06:27:11 +00:00
|
|
|
}
|
|
|
|
|
2023-02-15 01:06:56 +00:00
|
|
|
contract.Assertf(info.Path == filepath.Dir(path),
|
|
|
|
"plugin executable (%v) is not inside plugin directory (%v)", path, info.Path)
|
2022-07-22 09:33:59 +00:00
|
|
|
return path, err
|
2022-06-14 06:27:11 +00:00
|
|
|
}
|
|
|
|
|
2023-08-07 12:15:57 +00:00
|
|
|
func GetPluginInfo(d diag.Sink, kind PluginKind, name string, version *semver.Version,
|
2023-03-03 16:36:39 +00:00
|
|
|
projectPlugins []ProjectPlugin,
|
|
|
|
) (*PluginInfo, error) {
|
2023-08-07 12:15:57 +00:00
|
|
|
info, path, err := getPluginInfoAndPath(d, kind, name, version, false, projectPlugins)
|
2022-06-14 06:27:11 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2023-02-15 01:06:56 +00:00
|
|
|
contract.Assertf(info.Path == filepath.Dir(path),
|
|
|
|
"plugin executable (%v) is not inside plugin directory (%v)", path, info.Path)
|
2022-06-14 06:27:11 +00:00
|
|
|
return info, nil
|
|
|
|
}
|
|
|
|
|
2022-09-21 19:32:20 +00:00
|
|
|
// Given a PluginInfo try to find the executable file that corresponds to it
|
|
|
|
func getPluginPath(info *PluginInfo) string {
|
|
|
|
var path string
|
|
|
|
exts := getCandidateExtensions()
|
|
|
|
for _, ext := range exts {
|
|
|
|
path = filepath.Join(info.Path, info.Spec().File()) + ext
|
|
|
|
_, err := os.Stat(path)
|
|
|
|
if err == nil {
|
|
|
|
return path
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// We didn't actually find a file for this plugin, so just use the old behaviour of assuming the first
|
|
|
|
// extension.
|
|
|
|
return filepath.Join(info.Path, info.Spec().File()) + exts[0]
|
|
|
|
}
|
|
|
|
|
2022-07-22 09:33:59 +00:00
|
|
|
// getPluginInfoAndPath searches for a compatible plugin kind, name, and version and returns either:
|
2022-09-14 02:12:02 +00:00
|
|
|
// - if found as an ambient plugin, nil and the path to the executable
|
|
|
|
// - if found in the pulumi dir's installed plugins, a PluginInfo and path to the executable
|
|
|
|
// - an error in all other cases.
|
2022-07-22 09:33:59 +00:00
|
|
|
func getPluginInfoAndPath(
|
2023-08-07 12:15:57 +00:00
|
|
|
d diag.Sink,
|
2022-07-22 13:17:43 +00:00
|
|
|
kind PluginKind, name string, version *semver.Version, skipMetadata bool,
|
2023-03-03 16:36:39 +00:00
|
|
|
projectPlugins []ProjectPlugin,
|
|
|
|
) (*PluginInfo, string, error) {
|
2023-07-18 14:21:44 +00:00
|
|
|
filename := (&PluginSpec{Kind: kind, Name: name}).File()
|
2021-05-04 15:30:59 +00:00
|
|
|
|
2022-07-22 13:17:43 +00:00
|
|
|
for i, p1 := range projectPlugins {
|
|
|
|
for j, p2 := range projectPlugins {
|
|
|
|
if j < i {
|
|
|
|
if p2.Kind == p1.Kind && p2.Name == p1.Name {
|
|
|
|
if p1.Version != nil && p2.Version != nil && p2.Version.Equals(*p1.Version) {
|
|
|
|
return nil, "", fmt.Errorf(
|
|
|
|
"multiple project plugins with kind %s, name %s, version %s",
|
|
|
|
p1.Kind, p1.Name, p1.Version)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, plugin := range projectPlugins {
|
|
|
|
if plugin.Kind != kind {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if plugin.Name != name {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if plugin.Version != nil && version != nil {
|
|
|
|
if !plugin.Version.Equals(*version) {
|
|
|
|
logging.Warningf(
|
|
|
|
"Project plugin %s with version %s is incompatible with requested version %s.\n",
|
|
|
|
name, plugin.Version, version)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-08-26 14:51:14 +00:00
|
|
|
spec := plugin.Spec()
|
2022-08-18 14:31:10 +00:00
|
|
|
info := &PluginInfo{
|
2022-08-26 14:51:14 +00:00
|
|
|
Name: spec.Name,
|
|
|
|
Kind: spec.Kind,
|
|
|
|
Version: spec.Version,
|
2022-08-18 14:31:10 +00:00
|
|
|
Path: plugin.Path,
|
|
|
|
}
|
2022-09-21 19:32:20 +00:00
|
|
|
path := getPluginPath(info)
|
2022-08-26 14:51:14 +00:00
|
|
|
// computing plugin sizes can be very expensive (nested node_modules)
|
2022-09-21 19:32:20 +00:00
|
|
|
if !skipMetadata && path != "" {
|
2022-08-26 14:51:14 +00:00
|
|
|
if err := info.SetFileMetadata(path); err != nil {
|
|
|
|
return nil, "", err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return info, path, nil
|
2022-07-22 13:17:43 +00:00
|
|
|
}
|
|
|
|
|
2021-05-04 15:30:59 +00:00
|
|
|
// If we have a version of the plugin on its $PATH, use it, unless we have opted out of this behavior explicitly.
|
|
|
|
// This supports development scenarios.
|
2023-06-02 16:45:01 +00:00
|
|
|
includeAmbient := !(env.IgnoreAmbientPlugins.Value())
|
2023-08-07 12:15:57 +00:00
|
|
|
var ambientPath string
|
2022-03-17 12:22:56 +00:00
|
|
|
if includeAmbient {
|
2021-05-04 15:30:59 +00:00
|
|
|
if path, err := exec.LookPath(filename); err == nil {
|
2023-08-07 12:15:57 +00:00
|
|
|
ambientPath = path
|
2021-05-04 15:30:59 +00:00
|
|
|
logging.V(6).Infof("GetPluginPath(%s, %s, %v): found on $PATH %s", kind, name, version, path)
|
|
|
|
}
|
2018-04-05 23:37:50 +00:00
|
|
|
}
|
2018-02-06 15:20:04 +00:00
|
|
|
|
2022-04-14 11:35:12 +00:00
|
|
|
// At some point in the future, bundled plugins will be located in the plugin cache, just like regular
|
|
|
|
// plugins (see pulumi/pulumi#956 for some of the reasons why this isn't the case today). For now, they
|
|
|
|
// ship next to the `pulumi` binary. While we encourage this folder to be on the $PATH (and so the check
|
2023-06-02 16:45:01 +00:00
|
|
|
// above would have normally found the plugin) it's possible someone is running `pulumi` with an explicit
|
|
|
|
// path on the command line or has done symlink magic such that `pulumi` is on the path, but the bundled
|
|
|
|
// plugins are not, or has simply set IGNORE_AMBIENT_PLUGINS. So, if possible, look next to the instance
|
|
|
|
// of `pulumi` that is running to find this bundled plugin.
|
2023-08-07 12:15:57 +00:00
|
|
|
var bundledPath string
|
2023-03-31 18:33:55 +00:00
|
|
|
if IsPluginBundled(kind, name) {
|
Look for language plugins next to `pulumi` when probing
When launching plugins today, `pulumi` looks in two places:
1. It looks to see if the plugin in on the $PATH and if so, uses
it. This makes it easy to force a specific version of a resource
provider to be used and is what happens at development time (since
resource providers make their way onto $PATH via GOBIN).
2. If the above fails, it looks in the "plugin cache" in
`~/.pulumi/plugins`. This is the location that `pulumi plugin
install` places plugins.
Unlike resource provider plugins, we don't yet deliver language
plugins via `pulumi plugin install` so the language provider plugins
must be on the `$PATH` to be found. This is okay, because when we ship
the SDK, we include the executables next to `pulumi` itself.
However, if a user chooses to not put `pulumi` on their $PATH, or they
do but it is a symlink to the real `pulumi` binary installed
somewhere, we'd fail to find the language plugins, since they would
not be on the `$PATH`
To address this, when probing for language plugins, also consider
binaries next to the currently running `pulumi` process.
Fixes #1956
2018-11-19 23:07:01 +00:00
|
|
|
exePath, exeErr := os.Executable()
|
|
|
|
if exeErr == nil {
|
|
|
|
fullPath, fullErr := filepath.EvalSymlinks(exePath)
|
|
|
|
if fullErr == nil {
|
|
|
|
for _, ext := range getCandidateExtensions() {
|
|
|
|
candidate := filepath.Join(filepath.Dir(fullPath), filename+ext)
|
|
|
|
// Let's see if the file is executable. On Windows, os.Stat() returns a mode of "-rw-rw-rw" so on
|
|
|
|
// on windows we just trust the fact that the .exe can actually be launched.
|
|
|
|
if stat, err := os.Stat(candidate); err == nil &&
|
2023-03-03 16:36:39 +00:00
|
|
|
(stat.Mode()&0o100 != 0 || runtime.GOOS == windowsGOOS) {
|
Look for language plugins next to `pulumi` when probing
When launching plugins today, `pulumi` looks in two places:
1. It looks to see if the plugin in on the $PATH and if so, uses
it. This makes it easy to force a specific version of a resource
provider to be used and is what happens at development time (since
resource providers make their way onto $PATH via GOBIN).
2. If the above fails, it looks in the "plugin cache" in
`~/.pulumi/plugins`. This is the location that `pulumi plugin
install` places plugins.
Unlike resource provider plugins, we don't yet deliver language
plugins via `pulumi plugin install` so the language provider plugins
must be on the `$PATH` to be found. This is okay, because when we ship
the SDK, we include the executables next to `pulumi` itself.
However, if a user chooses to not put `pulumi` on their $PATH, or they
do but it is a symlink to the real `pulumi` binary installed
somewhere, we'd fail to find the language plugins, since they would
not be on the `$PATH`
To address this, when probing for language plugins, also consider
binaries next to the currently running `pulumi` process.
Fixes #1956
2018-11-19 23:07:01 +00:00
|
|
|
logging.V(6).Infof("GetPluginPath(%s, %s, %v): found next to current executable %s",
|
|
|
|
kind, name, version, candidate)
|
2023-08-07 12:15:57 +00:00
|
|
|
bundledPath = candidate
|
|
|
|
break
|
Look for language plugins next to `pulumi` when probing
When launching plugins today, `pulumi` looks in two places:
1. It looks to see if the plugin in on the $PATH and if so, uses
it. This makes it easy to force a specific version of a resource
provider to be used and is what happens at development time (since
resource providers make their way onto $PATH via GOBIN).
2. If the above fails, it looks in the "plugin cache" in
`~/.pulumi/plugins`. This is the location that `pulumi plugin
install` places plugins.
Unlike resource provider plugins, we don't yet deliver language
plugins via `pulumi plugin install` so the language provider plugins
must be on the `$PATH` to be found. This is okay, because when we ship
the SDK, we include the executables next to `pulumi` itself.
However, if a user chooses to not put `pulumi` on their $PATH, or they
do but it is a symlink to the real `pulumi` binary installed
somewhere, we'd fail to find the language plugins, since they would
not be on the `$PATH`
To address this, when probing for language plugins, also consider
binaries next to the currently running `pulumi` process.
Fixes #1956
2018-11-19 23:07:01 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-08-07 12:15:57 +00:00
|
|
|
// We prefer the ambient path, but we need to check if this is the same as the bundled
|
|
|
|
// path to decide if we're warning or not.
|
|
|
|
pluginPath := bundledPath
|
|
|
|
if ambientPath != "" {
|
|
|
|
if ambientPath != bundledPath {
|
|
|
|
// They don't match _but_ it might be they just don't match because the pulumi install is symlinked,
|
|
|
|
// e.g. /opt/homebrew/bin/pulumi-language-nodejs -> /opt/homebrew/Cellar/pulumi/3.77.0/bin/pulumi-language-nodejs
|
|
|
|
// So before we warn, lets just check if we can resolve symlinks in the ambient path and then check again.
|
|
|
|
fullAmbientPath, err := filepath.EvalSymlinks(ambientPath)
|
|
|
|
// N.B, that we don't _return_ the resolved path, we return the original path. Also if resolving
|
|
|
|
// hits any errors then we just skip this warning, better to not warn than to error in a new way.
|
|
|
|
if err == nil {
|
|
|
|
if fullAmbientPath != bundledPath {
|
|
|
|
d.Warningf(diag.Message("", "using %s from $PATH at %s"), filename, ambientPath)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
pluginPath = ambientPath
|
|
|
|
}
|
|
|
|
if pluginPath != "" {
|
|
|
|
return &PluginInfo{
|
|
|
|
Kind: kind,
|
|
|
|
Name: name,
|
|
|
|
Path: filepath.Dir(pluginPath),
|
|
|
|
}, pluginPath, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Wasn't ambient, and wasn't bundled, so now check the plugin cache.
|
2022-06-14 06:27:11 +00:00
|
|
|
var plugins []PluginInfo
|
|
|
|
var err error
|
|
|
|
if skipMetadata {
|
|
|
|
plugins, err = GetPlugins()
|
|
|
|
} else {
|
|
|
|
plugins, err = GetPluginsWithMetadata()
|
|
|
|
}
|
2018-04-05 23:37:50 +00:00
|
|
|
if err != nil {
|
2023-01-19 20:22:54 +00:00
|
|
|
return nil, "", fmt.Errorf("loading plugin list: %w", err)
|
2018-04-05 23:37:50 +00:00
|
|
|
}
|
2019-03-01 23:42:38 +00:00
|
|
|
|
2018-04-05 23:37:50 +00:00
|
|
|
var match *PluginInfo
|
2019-03-01 23:42:38 +00:00
|
|
|
if !enableLegacyPluginBehavior && version != nil {
|
|
|
|
logging.V(6).Infof("GetPluginPath(%s, %s, %s): enabling new plugin behavior", kind, name, version)
|
2023-12-03 09:15:07 +00:00
|
|
|
match = SelectCompatiblePlugin(plugins, kind, name, semver.MustParseRange(version.String()))
|
2019-03-01 23:42:38 +00:00
|
|
|
} else {
|
2023-12-03 09:15:07 +00:00
|
|
|
logging.V(6).Infof("GetPluginPath(%s, %s, %s): using legacy plugin behavior", kind, name, version)
|
|
|
|
match = LegacySelectCompatiblePlugin(plugins, kind, name, version)
|
2018-02-19 18:58:03 +00:00
|
|
|
}
|
2018-02-24 00:01:44 +00:00
|
|
|
|
2018-04-05 23:37:50 +00:00
|
|
|
if match != nil {
|
2022-09-21 19:32:20 +00:00
|
|
|
matchPath := getPluginPath(match)
|
2018-05-15 22:28:00 +00:00
|
|
|
logging.V(6).Infof("GetPluginPath(%s, %s, %v): found in cache at %s", kind, name, version, matchPath)
|
2022-06-14 06:27:11 +00:00
|
|
|
return match, matchPath, nil
|
2018-02-19 17:31:00 +00:00
|
|
|
}
|
|
|
|
|
2023-03-15 15:24:39 +00:00
|
|
|
return nil, "", NewMissingError(kind, name, version, includeAmbient)
|
2018-02-06 15:20:04 +00:00
|
|
|
}
|
|
|
|
|
2019-03-26 20:29:34 +00:00
|
|
|
// SortedPluginInfo is a wrapper around PluginInfo that allows for sorting by version.
|
|
|
|
type SortedPluginInfo []PluginInfo
|
2019-03-01 23:42:38 +00:00
|
|
|
|
2019-03-26 20:29:34 +00:00
|
|
|
func (sp SortedPluginInfo) Len() int { return len(sp) }
|
|
|
|
func (sp SortedPluginInfo) Less(i, j int) bool {
|
2019-03-01 23:42:38 +00:00
|
|
|
iVersion := sp[i].Version
|
|
|
|
jVersion := sp[j].Version
|
|
|
|
switch {
|
|
|
|
case iVersion == nil && jVersion == nil:
|
|
|
|
return false
|
|
|
|
case iVersion == nil:
|
|
|
|
return true
|
|
|
|
case jVersion == nil:
|
|
|
|
return false
|
2022-06-02 19:55:16 +00:00
|
|
|
case iVersion.EQ(*jVersion):
|
|
|
|
return iVersion.String() < jVersion.String()
|
2019-03-01 23:42:38 +00:00
|
|
|
default:
|
|
|
|
return iVersion.LT(*jVersion)
|
|
|
|
}
|
|
|
|
}
|
2019-03-26 20:29:34 +00:00
|
|
|
func (sp SortedPluginInfo) Swap(i, j int) { sp[i], sp[j] = sp[j], sp[i] }
|
2019-03-01 23:42:38 +00:00
|
|
|
|
2022-08-26 14:51:14 +00:00
|
|
|
// SortedPluginSpec is a wrapper around PluginSpec that allows for sorting by version.
|
|
|
|
type SortedPluginSpec []PluginSpec
|
|
|
|
|
|
|
|
func (sp SortedPluginSpec) Len() int { return len(sp) }
|
|
|
|
func (sp SortedPluginSpec) Less(i, j int) bool {
|
|
|
|
iVersion := sp[i].Version
|
|
|
|
jVersion := sp[j].Version
|
|
|
|
switch {
|
|
|
|
case iVersion == nil && jVersion == nil:
|
|
|
|
return false
|
|
|
|
case iVersion == nil:
|
|
|
|
return true
|
|
|
|
case jVersion == nil:
|
|
|
|
return false
|
|
|
|
case iVersion.EQ(*jVersion):
|
|
|
|
return iVersion.String() < jVersion.String()
|
|
|
|
default:
|
|
|
|
return iVersion.LT(*jVersion)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
func (sp SortedPluginSpec) Swap(i, j int) { sp[i], sp[j] = sp[j], sp[i] }
|
|
|
|
|
2023-12-03 09:15:07 +00:00
|
|
|
// LegacySelectCompatiblePlugin selects a plugin from the list of plugins with the given kind and name that
|
|
|
|
// satisfies the requested version. It returns the highest version plugin greater than the requested version,
|
|
|
|
// or an error if no such plugin could be found.
|
|
|
|
//
|
|
|
|
// If there exist plugins in the plugin list that don't have a version, LegacySelectCompatiblePlugin will select
|
|
|
|
// them if there are no other compatible plugins available.
|
|
|
|
func LegacySelectCompatiblePlugin(
|
|
|
|
plugins []PluginInfo, kind PluginKind, name string, version *semver.Version,
|
|
|
|
) *PluginInfo {
|
|
|
|
var match *PluginInfo
|
|
|
|
for _, cur := range plugins {
|
|
|
|
// Since the value of cur changes as we iterate, we can't save a pointer to it. So let's have a local
|
|
|
|
// that we can take a pointer to if this plugin is the best match yet.
|
|
|
|
plugin := cur
|
|
|
|
if plugin.Kind == kind && plugin.Name == name {
|
|
|
|
// Always pick the most recent version of the plugin available. Even if this is an exact match,
|
|
|
|
// we keep on searching just in case there's a newer version available.
|
|
|
|
var m *PluginInfo
|
|
|
|
if match == nil {
|
|
|
|
// no existing match
|
|
|
|
if version == nil {
|
|
|
|
m = &plugin // no version spec, accept anything
|
|
|
|
} else if plugin.Version == nil || plugin.Version.GTE(*version) {
|
|
|
|
// Either the plugin doesn't have a version, in which case we'll take it but prefer
|
|
|
|
// anything else, or it has a version >= requested.
|
|
|
|
m = &plugin
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// existing match
|
|
|
|
if plugin.Version != nil && match.Version == nil {
|
|
|
|
// existing match is unversioned, but this plugin has a version, so prefer it.
|
|
|
|
m = &plugin
|
|
|
|
} else if plugin.Version == nil {
|
|
|
|
// this plugin is unversioned ignore it, our current match might also be unversioned but
|
|
|
|
// we just pick the first we see in this case.
|
|
|
|
} else {
|
|
|
|
// both have versions, pick the greater stable one.
|
|
|
|
matchIsPre := len(match.Version.Pre) != 0
|
|
|
|
pluginIsPre := len(plugin.Version.Pre) != 0
|
|
|
|
|
|
|
|
// The plugin has to at least be greater than the requested version.
|
|
|
|
if version == nil || plugin.Version.GTE(*version) {
|
|
|
|
if matchIsPre && !pluginIsPre {
|
|
|
|
// If one is pre-release and the other is not, prefer the non-pre-release one.
|
|
|
|
m = &plugin
|
|
|
|
} else if !matchIsPre && pluginIsPre {
|
|
|
|
// current match is not pre-release, but this plugin is, so prefer the current match.
|
|
|
|
} else if plugin.Version.GT(*match.Version) {
|
|
|
|
// Else if the plugin is greater than the current match, prefer it.
|
|
|
|
m = &plugin
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if m != nil {
|
|
|
|
match = m
|
|
|
|
logging.V(6).Infof("LegacySelectCompatiblePlugin(%s, %s, %s): found candidate (#%s)",
|
|
|
|
kind, name, version, match.Version)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return match
|
|
|
|
}
|
|
|
|
|
2019-03-01 23:42:38 +00:00
|
|
|
// SelectCompatiblePlugin selects a plugin from the list of plugins with the given kind and name that sastisfies the
|
|
|
|
// requested semver range. It returns the highest version plugin that satisfies the requested constraints, or an error
|
|
|
|
// if no such plugin could be found.
|
|
|
|
//
|
|
|
|
// If there exist plugins in the plugin list that don't have a version, SelectCompatiblePlugin will select them if there
|
|
|
|
// are no other compatible plugins available.
|
|
|
|
func SelectCompatiblePlugin(
|
2023-03-03 16:36:39 +00:00
|
|
|
plugins []PluginInfo, kind PluginKind, name string, requested semver.Range,
|
2023-12-03 09:15:07 +00:00
|
|
|
) *PluginInfo {
|
2019-03-01 23:42:38 +00:00
|
|
|
logging.V(7).Infof("SelectCompatiblePlugin(..., %s): beginning", name)
|
|
|
|
var bestMatch PluginInfo
|
|
|
|
var hasMatch bool
|
|
|
|
|
|
|
|
// Before iterating over the list of plugins, sort the list of plugins by version in ascending order. This ensures
|
|
|
|
// that we can do a single pass over the plugin list, from lowest version to greatest version, and be confident that
|
|
|
|
// the best match that we find at the end is the greatest possible compatible version for the requested plugin.
|
|
|
|
//
|
|
|
|
// Plugins without versions are treated as having the lowest version. Ties between plugins without versions are
|
|
|
|
// resolved arbitrarily.
|
2019-03-26 20:29:34 +00:00
|
|
|
sort.Sort(SortedPluginInfo(plugins))
|
2019-03-01 23:42:38 +00:00
|
|
|
for _, plugin := range plugins {
|
|
|
|
switch {
|
|
|
|
case plugin.Kind != kind || plugin.Name != name:
|
|
|
|
// Not the plugin we're looking for.
|
|
|
|
case !hasMatch && plugin.Version == nil:
|
|
|
|
// This is the plugin we're looking for, but it doesn't have a version. We haven't seen anything better yet,
|
|
|
|
// so take it.
|
|
|
|
logging.V(7).Infof(
|
|
|
|
"SelectCompatiblePlugin(..., %s): best plugin %s: no version and no other candidates",
|
|
|
|
name, plugin.String())
|
|
|
|
hasMatch = true
|
|
|
|
bestMatch = plugin
|
|
|
|
case plugin.Version == nil:
|
|
|
|
// This is a rare case - we've already seen a version-less plugin and we're seeing another here. Ignore this
|
|
|
|
// one and defer to the one we previously selected.
|
|
|
|
logging.V(7).Infof("SelectCompatiblePlugin(..., %s): skipping plugin %s: no version", name, plugin.String())
|
|
|
|
case requested(*plugin.Version):
|
|
|
|
// This plugin is compatible with the requested semver range. Save it as the best match and continue.
|
|
|
|
logging.V(7).Infof("SelectCompatiblePlugin(..., %s): best plugin %s: semver match", name, plugin.String())
|
|
|
|
hasMatch = true
|
|
|
|
bestMatch = plugin
|
|
|
|
default:
|
|
|
|
logging.V(7).Infof(
|
|
|
|
"SelectCompatiblePlugin(..., %s): skipping plugin %s: semver mismatch", name, plugin.String())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if !hasMatch {
|
2023-12-03 09:15:07 +00:00
|
|
|
return nil
|
2019-03-01 23:42:38 +00:00
|
|
|
}
|
2023-12-03 09:15:07 +00:00
|
|
|
return &bestMatch
|
2019-03-01 23:42:38 +00:00
|
|
|
}
|
|
|
|
|
2019-08-21 12:22:41 +00:00
|
|
|
// ReadCloserProgressBar displays a progress bar for the given closer and returns a wrapper closer to manipulate it.
|
2019-08-21 18:47:58 +00:00
|
|
|
func ReadCloserProgressBar(
|
2023-03-03 16:36:39 +00:00
|
|
|
closer io.ReadCloser, size int64, message string, colorization colors.Colorization,
|
|
|
|
) io.ReadCloser {
|
2019-08-21 12:22:41 +00:00
|
|
|
if size == -1 {
|
|
|
|
return closer
|
|
|
|
}
|
|
|
|
|
2022-05-25 23:54:14 +00:00
|
|
|
if !cmdutil.Interactive() {
|
|
|
|
return closer
|
|
|
|
}
|
|
|
|
|
2019-08-21 12:22:41 +00:00
|
|
|
// If we know the length of the download, show a progress bar.
|
|
|
|
bar := pb.New(int(size))
|
2021-05-26 02:02:09 +00:00
|
|
|
bar.Output = os.Stderr
|
2019-08-21 18:34:23 +00:00
|
|
|
bar.Prefix(colorization.Colorize(colors.SpecUnimportant + message + ":"))
|
2019-08-21 12:22:41 +00:00
|
|
|
bar.Postfix(colorization.Colorize(colors.Reset))
|
|
|
|
bar.SetMaxWidth(80)
|
|
|
|
bar.SetUnits(pb.U_BYTES)
|
|
|
|
bar.Start()
|
|
|
|
|
|
|
|
return &barCloser{
|
|
|
|
bar: bar,
|
|
|
|
readCloser: bar.NewProxyReader(closer),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Look for language plugins next to `pulumi` when probing
When launching plugins today, `pulumi` looks in two places:
1. It looks to see if the plugin in on the $PATH and if so, uses
it. This makes it easy to force a specific version of a resource
provider to be used and is what happens at development time (since
resource providers make their way onto $PATH via GOBIN).
2. If the above fails, it looks in the "plugin cache" in
`~/.pulumi/plugins`. This is the location that `pulumi plugin
install` places plugins.
Unlike resource provider plugins, we don't yet deliver language
plugins via `pulumi plugin install` so the language provider plugins
must be on the `$PATH` to be found. This is okay, because when we ship
the SDK, we include the executables next to `pulumi` itself.
However, if a user chooses to not put `pulumi` on their $PATH, or they
do but it is a symlink to the real `pulumi` binary installed
somewhere, we'd fail to find the language plugins, since they would
not be on the `$PATH`
To address this, when probing for language plugins, also consider
binaries next to the currently running `pulumi` process.
Fixes #1956
2018-11-19 23:07:01 +00:00
|
|
|
// getCandidateExtensions returns a set of file extensions (including the dot seprator) which should be used when
|
|
|
|
// probing for an executable file.
|
|
|
|
func getCandidateExtensions() []string {
|
|
|
|
if runtime.GOOS == windowsGOOS {
|
|
|
|
return []string{".exe", ".cmd"}
|
|
|
|
}
|
|
|
|
|
|
|
|
return []string{""}
|
|
|
|
}
|
|
|
|
|
2019-09-03 20:40:44 +00:00
|
|
|
// pluginRegexp matches plugin directory names: pulumi-KIND-NAME-VERSION.
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
var pluginRegexp = regexp.MustCompile(
|
2018-02-19 17:31:00 +00:00
|
|
|
"^(?P<Kind>[a-z]+)-" + // KIND
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
"(?P<Name>[a-zA-Z0-9-]*[a-zA-Z0-9])-" + // NAME
|
2018-11-21 23:52:20 +00:00
|
|
|
"v(?P<Version>.*)$") // VERSION
|
2018-02-19 17:31:00 +00:00
|
|
|
|
Fix plugin install failures on Windows (#5759)
When installing a plugin, previous versions of Pulumi extracted the
plugin tarball to a temp directory and then renamed the temp directory
to the final plugin directory. This was done to prevent concurrent
installs: if a process fails to rename the temp dir because the final
dir already exists, it means another process already installed the
plugin. Unfortunately, on Windows the rename operation often fails due
to aggressive virus scanners opening files in the temp dir.
In order to provide reliable plugin installs on Windows, we now extract
the tarball directly into the final directory, and use file locks to
prevent concurrent installs from toppling over one another.
During install, a lock file is created in the plugin cache directory
with the same name as the plugin's final directory but suffixed with
`.lock`. The process that obtains the lock is responsible for extracting
the tarball. Before it does that, it cleans up any previous temp
directories of failed installs of previous versions of Pulumi. Then it
creates an empty `.partial` file next to the `.lock` file. The
`.partial` file indicates an installation is in-progress. The `.partial`
file is deleted when installation is complete, indicating the plugin was
successfully installed. If a failure occurs during installation, the
`.partial` file will remain indicating the plugin wasn't fully
installed. The next time the plugin is installed, the old installation
directory will be removed and replaced with a fresh install.
This is the same approach Go uses for installing modules in its
module cache.
2020-11-16 17:44:29 +00:00
|
|
|
// installingPluginRegexp matches the name of temporary folders. Previous versions of Pulumi first extracted
|
|
|
|
// plugins to a temporary folder with a suffix of `.tmpXXXXXX` (where `XXXXXX`) is a random number, from
|
2023-01-06 22:39:16 +00:00
|
|
|
// os.CreateTemp. We should ignore these folders.
|
2019-09-03 20:40:44 +00:00
|
|
|
var installingPluginRegexp = regexp.MustCompile(`\.tmp[0-9]+$`)
|
|
|
|
|
2018-02-19 17:31:00 +00:00
|
|
|
// tryPlugin returns true if a file is a plugin, and extracts information about it.
|
2022-09-10 07:22:11 +00:00
|
|
|
func tryPlugin(file os.DirEntry) (PluginKind, string, semver.Version, bool) {
|
2018-02-19 17:31:00 +00:00
|
|
|
// Only directories contain plugins.
|
|
|
|
if !file.IsDir() {
|
2018-05-15 22:28:00 +00:00
|
|
|
logging.V(11).Infof("skipping file in plugin directory: %s", file.Name())
|
2019-09-03 20:40:44 +00:00
|
|
|
return "", "", semver.Version{}, false
|
|
|
|
}
|
|
|
|
|
|
|
|
// Ignore plugins which are being installed
|
|
|
|
if installingPluginRegexp.MatchString(file.Name()) {
|
|
|
|
logging.V(11).Infof("skipping plugin %s which is being installed", file.Name())
|
2018-02-06 15:20:04 +00:00
|
|
|
return "", "", semver.Version{}, false
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Filenames must match the plugin regexp.
|
|
|
|
match := pluginRegexp.FindStringSubmatch(file.Name())
|
|
|
|
if len(match) != len(pluginRegexp.SubexpNames()) {
|
2018-05-15 22:28:00 +00:00
|
|
|
logging.V(11).Infof("skipping plugin %s with missing capture groups: expect=%d, actual=%d",
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
file.Name(), len(pluginRegexp.SubexpNames()), len(match))
|
2018-02-06 15:20:04 +00:00
|
|
|
return "", "", semver.Version{}, false
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
|
|
|
var kind PluginKind
|
|
|
|
var name string
|
|
|
|
var version *semver.Version
|
|
|
|
for i, group := range pluginRegexp.SubexpNames() {
|
|
|
|
v := match[i]
|
|
|
|
switch group {
|
|
|
|
case "Kind":
|
|
|
|
// Skip invalid kinds.
|
|
|
|
if IsPluginKind(v) {
|
|
|
|
kind = PluginKind(v)
|
|
|
|
} else {
|
2018-05-15 22:28:00 +00:00
|
|
|
logging.V(11).Infof("skipping invalid plugin kind: %s", v)
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
|
|
|
case "Name":
|
|
|
|
name = v
|
|
|
|
case "Version":
|
|
|
|
// Skip invalid versions.
|
|
|
|
ver, err := semver.ParseTolerant(v)
|
|
|
|
if err == nil {
|
|
|
|
version = &ver
|
|
|
|
} else {
|
2018-05-15 22:28:00 +00:00
|
|
|
logging.V(11).Infof("skipping invalid plugin version: %s", v)
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// If anything was missing or invalid, skip this plugin.
|
|
|
|
if kind == "" || name == "" || version == nil {
|
2018-05-15 22:28:00 +00:00
|
|
|
logging.V(11).Infof("skipping plugin with missing information: kind=%s, name=%s, version=%v",
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
kind, name, version)
|
2018-02-06 15:20:04 +00:00
|
|
|
return "", "", semver.Version{}, false
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
|
|
|
|
2018-02-06 15:20:04 +00:00
|
|
|
return kind, name, *version, true
|
Implement basic plugin management
This change implements basic plugin management, but we do not yet
actually use the plugins for anything (that comes next).
Plugins are stored in `~/.pulumi/plugins`, and are expected to be
in the format `pulumi-<KIND>-<NAME>-v<VERSION>[.exe]`. The KIND is
one of `analyzer`, `language`, or `resource`, the NAME is a hyphen-
delimited name (e.g., `aws` or `foo-bar`), and VERSION is the
plugin's semantic version (e.g., `0.9.11`, `1.3.7-beta.a736cf`, etc).
This commit includes four new CLI commands:
* `pulumi plugin` is the top-level plugin command. It does nothing
but show the help text for associated child commands.
* `pulumi plugin install` can be used to install plugins manually.
If run with no additional arguments, it will compute the set of
plugins used by the current project, and download them all. It
may be run to explicitly download a single plugin, however, by
invoking it as `pulumi plugin install KIND NAME VERSION`. For
example, `pulumi plugin install resource aws v0.9.11`. By default,
this command uses the cloud backend in the usual way to perform the
download, although a separate URL may be given with --cloud-url,
just like all other commands that interact with our backend service.
* `pulumi plugin ls` lists all plugins currently installed in the
plugin cache. It displays some useful statistics, like the size
of the plugin, when it was installed, when it was last used, and
so on. It sorts the display alphabetically by plugin name, and
for plugins with multiple versions, it shows the newest at the top.
The command also summarizes how much disk space is currently being
consumed by the plugin cache. There are no filtering capabilities yet.
* `pulumi plugin prune` will delete plugins from the cache. By
default, when run with no arguments, it will delete everything.
It may be run with additional arguments, KIND, NAME, and VERSION,
each one getting more specific about what it will delete. For
instance, `pulumi plugin prune resource aws` will delete all AWS
plugin versions, while `pulumi plugin prune resource aws <0.9`
will delete all AWS plugins before version 0.9. Unless --yes is
passed, the command will confirm the deletion with a count of how
many plugins will be affected by the command.
We do not yet actually download plugins on demand yet. That will
come in a subsequent change.
2018-02-04 18:51:29 +00:00
|
|
|
}
|
2018-02-19 17:31:00 +00:00
|
|
|
|
|
|
|
// getPluginSize recursively computes how much space is devoted to a given plugin.
|
2018-04-16 16:26:46 +00:00
|
|
|
func getPluginSize(path string) (int64, error) {
|
|
|
|
file, err := os.Stat(path)
|
2018-02-19 17:31:00 +00:00
|
|
|
if err != nil {
|
2018-04-16 16:26:46 +00:00
|
|
|
return 0, nil
|
2018-02-19 17:31:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
size := int64(0)
|
2018-04-16 16:26:46 +00:00
|
|
|
if file.IsDir() {
|
2022-09-10 07:22:11 +00:00
|
|
|
subs, err := os.ReadDir(path)
|
2018-04-16 16:26:46 +00:00
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
for _, child := range subs {
|
|
|
|
add, err := getPluginSize(filepath.Join(path, child.Name()))
|
2018-02-19 17:31:00 +00:00
|
|
|
if err != nil {
|
2018-04-16 16:26:46 +00:00
|
|
|
return 0, err
|
2018-02-19 17:31:00 +00:00
|
|
|
}
|
2018-04-16 16:26:46 +00:00
|
|
|
size += add
|
2018-02-19 17:31:00 +00:00
|
|
|
}
|
2018-04-16 16:26:46 +00:00
|
|
|
} else {
|
|
|
|
size += file.Size()
|
2018-02-19 17:31:00 +00:00
|
|
|
}
|
|
|
|
return size, nil
|
|
|
|
}
|
2019-08-21 12:22:41 +00:00
|
|
|
|
|
|
|
type barCloser struct {
|
|
|
|
bar *pb.ProgressBar
|
|
|
|
readCloser io.ReadCloser
|
|
|
|
}
|
|
|
|
|
|
|
|
func (bc *barCloser) Read(dest []byte) (int, error) {
|
|
|
|
return bc.readCloser.Read(dest)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (bc *barCloser) Close() error {
|
|
|
|
bc.bar.Finish()
|
|
|
|
return bc.readCloser.Close()
|
|
|
|
}
|