2023-08-31 16:35:21 +00:00
|
|
|
|
// Copyright 2016-2023, Pulumi Corporation.
|
2018-05-22 19:43:36 +00:00
|
|
|
|
//
|
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
|
//
|
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
//
|
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
|
// limitations under the License.
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
|
|
|
|
|
// pulumi-language-python serves as the "language host" for Pulumi programs written in Python. It is ultimately
|
|
|
|
|
// responsible for spawning the language runtime that executes the program.
|
|
|
|
|
//
|
|
|
|
|
// The program being executed is executed by a shim script called `pulumi-language-python-exec`. This script is
|
|
|
|
|
// written in the hosted language (in this case, Python) and is responsible for initiating RPC links to the resource
|
|
|
|
|
// monitor and engine.
|
|
|
|
|
//
|
|
|
|
|
// It's therefore the responsibility of this program to implement the LanguageHostServer endpoint by spawning
|
|
|
|
|
// instances of `pulumi-language-python-exec` and forwarding the RPC request arguments to the command-line.
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
|
|
import (
|
2021-08-25 15:40:58 +00:00
|
|
|
|
"bytes"
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
"context"
|
|
|
|
|
"encoding/json"
|
2023-01-03 10:19:35 +00:00
|
|
|
|
"errors"
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
"flag"
|
|
|
|
|
"fmt"
|
2020-12-04 03:22:16 +00:00
|
|
|
|
"math/rand"
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
"os"
|
|
|
|
|
"os/exec"
|
2023-08-29 15:42:31 +00:00
|
|
|
|
"os/signal"
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
"path/filepath"
|
2023-12-12 12:19:42 +00:00
|
|
|
|
"strconv"
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
"strings"
|
|
|
|
|
"syscall"
|
2022-06-06 12:28:00 +00:00
|
|
|
|
"time"
|
2022-02-10 10:25:53 +00:00
|
|
|
|
"unicode"
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
|
2021-11-09 22:22:16 +00:00
|
|
|
|
"github.com/blang/semver"
|
2021-03-17 13:20:05 +00:00
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
|
2023-06-28 16:02:04 +00:00
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/slice"
|
2021-03-17 13:20:05 +00:00
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
|
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
|
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/fsutil"
|
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
|
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
|
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/version"
|
2023-08-31 16:35:21 +00:00
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
|
2021-03-17 13:20:05 +00:00
|
|
|
|
pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
|
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/python"
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
"google.golang.org/grpc"
|
2023-01-11 19:54:31 +00:00
|
|
|
|
"google.golang.org/grpc/credentials/insecure"
|
2024-01-17 09:35:20 +00:00
|
|
|
|
"google.golang.org/protobuf/types/known/emptypb"
|
2023-08-31 16:35:21 +00:00
|
|
|
|
|
|
|
|
|
hclsyntax "github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax"
|
|
|
|
|
"github.com/pulumi/pulumi/pkg/v3/codegen/pcl"
|
|
|
|
|
codegen "github.com/pulumi/pulumi/pkg/v3/codegen/python"
|
|
|
|
|
"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
const (
|
|
|
|
|
// By convention, the executor is the name of the current program (pulumi-language-python) plus this suffix.
|
2018-06-15 05:16:43 +00:00
|
|
|
|
pythonDefaultExec = "pulumi-language-python-exec" // the exec shim for Pulumi to run Python programs.
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
|
|
|
|
|
// The runtime expects the config object to be saved to this environment variable.
|
|
|
|
|
pulumiConfigVar = "PULUMI_CONFIG"
|
2021-05-18 22:00:30 +00:00
|
|
|
|
|
|
|
|
|
// The runtime expects the array of secret config keys to be saved to this environment variable.
|
2023-01-06 00:07:45 +00:00
|
|
|
|
//nolint:gosec
|
2021-05-18 22:00:30 +00:00
|
|
|
|
pulumiConfigSecretKeysVar = "PULUMI_CONFIG_SECRET_KEYS"
|
2022-08-11 17:38:34 +00:00
|
|
|
|
|
|
|
|
|
// A exit-code we recognize when the python process exits. If we see this error, there's no
|
|
|
|
|
// need for us to print any additional error messages since the user already got a a good
|
|
|
|
|
// one they can handle.
|
|
|
|
|
pythonProcessExitedAfterShowingUserActionableMessage = 32
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
)
|
|
|
|
|
|
2021-11-09 22:22:16 +00:00
|
|
|
|
var (
|
|
|
|
|
// The minimum python version that Pulumi supports
|
2022-09-01 11:21:49 +00:00
|
|
|
|
minimumSupportedPythonVersion = semver.MustParse("3.7.0")
|
2021-11-09 22:22:16 +00:00
|
|
|
|
// Any version less then `eolPythonVersion` is EOL.
|
|
|
|
|
eolPythonVersion = semver.MustParse("3.7.0")
|
|
|
|
|
// An url to the issue discussing EOL.
|
|
|
|
|
eolPythonVersionIssue = "https://github.com/pulumi/pulumi/issues/8131"
|
|
|
|
|
)
|
|
|
|
|
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
// Launches the language host RPC endpoint, which in turn fires up an RPC server implementing the
|
|
|
|
|
// LanguageRuntimeServer RPC endpoint.
|
|
|
|
|
func main() {
|
|
|
|
|
var tracing string
|
|
|
|
|
flag.StringVar(&tracing, "tracing", "", "Emit tracing to a Zipkin-compatible tracing endpoint")
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
flag.String("virtualenv", "", "[obsolete] Virtual environment path to use")
|
|
|
|
|
flag.String("root", "", "[obsolete] Project root path to use")
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
|
|
|
|
|
// You can use the below flag to request that the language host load a specific executor instead of probing the
|
|
|
|
|
// PATH. This can be used during testing to override the default location.
|
|
|
|
|
var givenExecutor string
|
|
|
|
|
flag.StringVar(&givenExecutor, "use-executor", "",
|
|
|
|
|
"Use the given program as the executor instead of looking for one on PATH")
|
|
|
|
|
|
|
|
|
|
flag.Parse()
|
|
|
|
|
args := flag.Args()
|
2018-05-15 22:28:00 +00:00
|
|
|
|
logging.InitLogging(false, 0, false)
|
2018-05-08 01:23:03 +00:00
|
|
|
|
cmdutil.InitTracing("pulumi-language-python", "pulumi-language-python", tracing)
|
2018-06-15 05:16:43 +00:00
|
|
|
|
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
var pythonExec string
|
|
|
|
|
if givenExecutor == "" {
|
2018-06-15 05:16:43 +00:00
|
|
|
|
// By default, the -exec script is installed next to the language host.
|
|
|
|
|
thisPath, err := os.Executable()
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
if err != nil {
|
2023-01-03 10:19:35 +00:00
|
|
|
|
err = fmt.Errorf("could not determine current executable: %w", err)
|
2018-06-15 05:16:43 +00:00
|
|
|
|
cmdutil.Exit(err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pathExec := filepath.Join(filepath.Dir(thisPath), pythonDefaultExec)
|
|
|
|
|
if _, err = os.Stat(pathExec); os.IsNotExist(err) {
|
2023-01-03 10:19:35 +00:00
|
|
|
|
err = fmt.Errorf("missing executor %s: %w", pathExec, err)
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
cmdutil.Exit(err)
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-15 22:28:00 +00:00
|
|
|
|
logging.V(3).Infof("language host identified executor from path: `%s`", pathExec)
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
pythonExec = pathExec
|
|
|
|
|
} else {
|
2018-05-15 22:28:00 +00:00
|
|
|
|
logging.V(3).Infof("language host asked to use specific executor: `%s`", givenExecutor)
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
pythonExec = givenExecutor
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Optionally pluck out the engine so we can do logging, etc.
|
|
|
|
|
var engineAddress string
|
|
|
|
|
if len(args) > 0 {
|
|
|
|
|
engineAddress = args[0]
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-29 15:42:31 +00:00
|
|
|
|
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
|
2022-06-06 12:28:00 +00:00
|
|
|
|
// map the context Done channel to the rpcutil boolean cancel channel
|
|
|
|
|
cancelChannel := make(chan bool)
|
|
|
|
|
go func() {
|
|
|
|
|
<-ctx.Done()
|
2023-08-29 15:42:31 +00:00
|
|
|
|
cancel() // deregister signal handler
|
2022-06-06 12:28:00 +00:00
|
|
|
|
close(cancelChannel)
|
|
|
|
|
}()
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
err := rpcutil.Healthcheck(ctx, engineAddress, 5*time.Minute, cancel)
|
2022-06-06 12:28:00 +00:00
|
|
|
|
if err != nil {
|
2023-01-03 10:19:35 +00:00
|
|
|
|
cmdutil.Exit(fmt.Errorf("could not start health check host RPC server: %w", err))
|
2022-06-06 12:28:00 +00:00
|
|
|
|
}
|
|
|
|
|
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
// Fire up a gRPC server, letting the kernel choose a free port.
|
2022-11-01 15:15:09 +00:00
|
|
|
|
handle, err := rpcutil.ServeWithOptions(rpcutil.ServeOptions{
|
|
|
|
|
Cancel: cancelChannel,
|
|
|
|
|
Init: func(srv *grpc.Server) error {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
host := newLanguageHost(pythonExec, engineAddress, tracing)
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
pulumirpc.RegisterLanguageRuntimeServer(srv, host)
|
|
|
|
|
return nil
|
|
|
|
|
},
|
2022-11-01 15:15:09 +00:00
|
|
|
|
Options: rpcutil.OpenTracingServerInterceptorOptions(nil),
|
|
|
|
|
})
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
if err != nil {
|
2023-01-03 10:19:35 +00:00
|
|
|
|
cmdutil.Exit(fmt.Errorf("could not start language host RPC server: %w", err))
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Otherwise, print out the port so that the spawner knows how to reach us.
|
2022-11-01 15:15:09 +00:00
|
|
|
|
fmt.Printf("%d\n", handle.Port)
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
|
|
|
|
|
// And finally wait for the server to stop serving.
|
2022-11-01 15:15:09 +00:00
|
|
|
|
if err := <-handle.Done; err != nil {
|
2023-01-03 10:19:35 +00:00
|
|
|
|
cmdutil.Exit(fmt.Errorf("language host RPC stopped serving: %w", err))
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// pythonLanguageHost implements the LanguageRuntimeServer interface
|
|
|
|
|
// for use as an API endpoint.
|
|
|
|
|
type pythonLanguageHost struct {
|
2022-12-14 19:20:26 +00:00
|
|
|
|
pulumirpc.UnimplementedLanguageRuntimeServer
|
|
|
|
|
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
exec string
|
|
|
|
|
engineAddress string
|
|
|
|
|
tracing string
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
}
|
2021-05-14 17:41:55 +00:00
|
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
type pythonOptions struct {
|
|
|
|
|
// Virtual environment path to use.
|
2021-05-14 17:41:55 +00:00
|
|
|
|
virtualenv string
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
// The resolved virtual environment path.
|
2021-05-14 17:41:55 +00:00
|
|
|
|
virtualenvPath string
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
}
|
|
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
func parseOptions(root string, options map[string]interface{}) (pythonOptions, error) {
|
|
|
|
|
var pythonOptions pythonOptions
|
|
|
|
|
if virtualenv, ok := options["virtualenv"]; ok {
|
|
|
|
|
if virtualenv, ok := virtualenv.(string); ok {
|
|
|
|
|
pythonOptions.virtualenv = virtualenv
|
|
|
|
|
} else {
|
|
|
|
|
return pythonOptions, errors.New("virtualenv option must be a string")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Resolve virtualenv path relative to root.
|
|
|
|
|
pythonOptions.virtualenvPath = resolveVirtualEnvironmentPath(root, pythonOptions.virtualenv)
|
|
|
|
|
|
|
|
|
|
return pythonOptions, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func newLanguageHost(exec, engineAddress, tracing string,
|
2023-03-03 16:36:39 +00:00
|
|
|
|
) pulumirpc.LanguageRuntimeServer {
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
return &pythonLanguageHost{
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
exec: exec,
|
|
|
|
|
engineAddress: engineAddress,
|
|
|
|
|
tracing: tracing,
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// GetRequiredPlugins computes the complete set of anticipated plugins required by a program.
|
|
|
|
|
func (host *pythonLanguageHost) GetRequiredPlugins(ctx context.Context,
|
2023-03-03 16:36:39 +00:00
|
|
|
|
req *pulumirpc.GetRequiredPluginsRequest,
|
|
|
|
|
) (*pulumirpc.GetRequiredPluginsResponse, error) {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
opts, err := parseOptions(req.Info.RootDirectory, req.Info.Options.AsMap())
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-04 03:22:16 +00:00
|
|
|
|
// Prepare the virtual environment (if needed).
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
err = host.prepareVirtualEnvironment(ctx, req.Info.ProgramDirectory, opts.virtualenvPath)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
if err != nil {
|
2024-01-30 15:02:59 +00:00
|
|
|
|
return nil, fmt.Errorf("preparing virtual environment: %w", err)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
validateVersion(ctx, opts.virtualenvPath)
|
2022-08-31 19:37:03 +00:00
|
|
|
|
|
2020-12-04 03:22:16 +00:00
|
|
|
|
// Now, determine which Pulumi packages are installed.
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
pulumiPackages, err := determinePulumiPackages(ctx, opts.virtualenvPath, req.Info.ProgramDirectory)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
plugins := []*pulumirpc.PluginDependency{}
|
|
|
|
|
for _, pkg := range pulumiPackages {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
plugin, err := determinePluginDependency(opts.virtualenvPath, req.Info.ProgramDirectory, pkg)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if plugin != nil {
|
|
|
|
|
plugins = append(plugins, plugin)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return &pulumirpc.GetRequiredPluginsResponse{Plugins: plugins}, nil
|
|
|
|
|
}
|
|
|
|
|
|
2024-01-30 15:02:59 +00:00
|
|
|
|
func (host *pythonLanguageHost) Pack(ctx context.Context, req *pulumirpc.PackRequest) (*pulumirpc.PackResponse, error) {
|
|
|
|
|
// ensure build is up-to-date
|
|
|
|
|
buildUpgradeCmd, err := python.Command(ctx, "-m", "pip", "install", "--upgrade", "build")
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
buildUpgradeCmd.Stdout = os.Stdout
|
|
|
|
|
buildUpgradeCmd.Stderr = os.Stderr
|
|
|
|
|
err = buildUpgradeCmd.Run()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, fmt.Errorf("install build tools: %w", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
tmp, err := os.MkdirTemp("", "pulumi-python-pack")
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, fmt.Errorf("create temporary directory: %w", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// translate the semver Version into a PEP440 version
|
|
|
|
|
version := "0.0.0"
|
|
|
|
|
if v, err := semver.Parse(req.Version); err == nil {
|
|
|
|
|
version = codegen.PypiVersion(v)
|
|
|
|
|
} else if req.Version != "" {
|
|
|
|
|
return nil, fmt.Errorf("invalid version: %w", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
buildCmd, err := python.Command(ctx, "-m", "build", "--outdir", tmp)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
buildCmd.Dir = req.PackageDirectory
|
|
|
|
|
buildCmd.Env = os.Environ()
|
|
|
|
|
buildCmd.Env = append(buildCmd.Env, "PULUMI_PYTHON_VERSION="+version)
|
|
|
|
|
|
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
|
|
|
buildCmd.Stdout = &stdout
|
|
|
|
|
buildCmd.Stderr = &stderr
|
|
|
|
|
|
|
|
|
|
err = buildCmd.Run()
|
|
|
|
|
logging.V(5).Infof("Pack stdout: %s", stdout.String())
|
|
|
|
|
logging.V(5).Infof("Pack stderr: %s", stderr.String())
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, fmt.Errorf("run python build: %w\n%s\n%s", err, stdout.String(), stderr.String())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// prefer .whl but return .tar.gz if no .whl is found
|
|
|
|
|
files, err := os.ReadDir(tmp)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, fmt.Errorf("read temporary directory: %w", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var found string
|
|
|
|
|
for _, file := range files {
|
|
|
|
|
if strings.HasSuffix(file.Name(), ".whl") {
|
|
|
|
|
found = file.Name()
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
if strings.HasSuffix(file.Name(), ".tar.gz") {
|
|
|
|
|
found = file.Name()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Copy the found file to the destination directory
|
|
|
|
|
if found == "" {
|
|
|
|
|
return nil, fmt.Errorf("no .whl or .tar.gz file found\n%s", stderr.String())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
src := filepath.Join(tmp, found)
|
|
|
|
|
dst := filepath.Join(req.DestinationDirectory, found)
|
|
|
|
|
err = fsutil.CopyFile(dst, src, nil)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, fmt.Errorf("copy file: %w", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return &pulumirpc.PackResponse{
|
|
|
|
|
ArtifactPath: dst,
|
|
|
|
|
}, nil
|
|
|
|
|
}
|
|
|
|
|
|
2021-05-14 17:41:55 +00:00
|
|
|
|
func resolveVirtualEnvironmentPath(root, virtualenv string) string {
|
2020-12-04 03:22:16 +00:00
|
|
|
|
if virtualenv == "" {
|
2021-05-14 17:41:55 +00:00
|
|
|
|
return ""
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
2021-03-24 19:51:46 +00:00
|
|
|
|
if !filepath.IsAbs(virtualenv) {
|
2021-05-14 17:41:55 +00:00
|
|
|
|
return filepath.Join(root, virtualenv)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
2021-05-14 17:41:55 +00:00
|
|
|
|
return virtualenv
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// prepareVirtualEnvironment will create and install dependencies in the virtual environment if host.virtualenv is set.
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
func (host *pythonLanguageHost) prepareVirtualEnvironment(ctx context.Context, pwd, virtualenv string) error {
|
|
|
|
|
if virtualenv == "" {
|
2021-05-14 17:41:55 +00:00
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-04 03:22:16 +00:00
|
|
|
|
// If the virtual environment directory doesn't exist, create it.
|
|
|
|
|
var createVirtualEnv bool
|
|
|
|
|
info, err := os.Stat(virtualenv)
|
|
|
|
|
if err != nil {
|
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
|
createVirtualEnv = true
|
|
|
|
|
} else {
|
2021-05-14 17:41:55 +00:00
|
|
|
|
return err
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
} else if !info.IsDir() {
|
2023-01-03 10:19:35 +00:00
|
|
|
|
return fmt.Errorf("the 'virtualenv' option in Pulumi.yaml is set to %q but it is not a directory", virtualenv)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// If the virtual environment directory exists, but is empty, it needs to be created.
|
|
|
|
|
if !createVirtualEnv {
|
|
|
|
|
empty, err := fsutil.IsDirEmpty(virtualenv)
|
|
|
|
|
if err != nil {
|
2021-05-14 17:41:55 +00:00
|
|
|
|
return err
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
createVirtualEnv = empty
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Create the virtual environment and install dependencies into it, if needed.
|
|
|
|
|
if createVirtualEnv {
|
|
|
|
|
// Make a connection to the real engine that we will log messages to.
|
|
|
|
|
conn, err := grpc.Dial(
|
|
|
|
|
host.engineAddress,
|
2023-01-11 19:54:31 +00:00
|
|
|
|
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
2020-12-04 03:22:16 +00:00
|
|
|
|
rpcutil.GrpcChannelOptions(),
|
|
|
|
|
)
|
|
|
|
|
if err != nil {
|
2023-01-03 10:19:35 +00:00
|
|
|
|
return fmt.Errorf("language host could not make connection to engine: %w", err)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Make a client around that connection.
|
|
|
|
|
engineClient := pulumirpc.NewEngineClient(conn)
|
|
|
|
|
|
|
|
|
|
// Create writers that log the output of the install operation as ephemeral messages.
|
|
|
|
|
streamID := rand.Int31() //nolint:gosec
|
|
|
|
|
|
|
|
|
|
infoWriter := &logWriter{
|
|
|
|
|
ctx: ctx,
|
|
|
|
|
engineClient: engineClient,
|
|
|
|
|
streamID: streamID,
|
|
|
|
|
severity: pulumirpc.LogSeverity_INFO,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
errorWriter := &logWriter{
|
|
|
|
|
ctx: ctx,
|
|
|
|
|
engineClient: engineClient,
|
|
|
|
|
streamID: streamID,
|
|
|
|
|
severity: pulumirpc.LogSeverity_ERROR,
|
|
|
|
|
}
|
|
|
|
|
|
2022-06-09 21:57:56 +00:00
|
|
|
|
if err := python.InstallDependenciesWithWriters(ctx,
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
pwd, virtualenv, true /*showOutput*/, infoWriter, errorWriter); err != nil {
|
2024-01-30 15:02:59 +00:00
|
|
|
|
return fmt.Errorf("installing dependencies: %w", err)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Ensure the specified virtual directory is a valid virtual environment.
|
|
|
|
|
if !python.IsVirtualEnv(virtualenv) {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
return python.NewVirtualEnvError(virtualenv, virtualenv)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
2021-05-14 17:41:55 +00:00
|
|
|
|
return nil
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type logWriter struct {
|
|
|
|
|
ctx context.Context
|
|
|
|
|
engineClient pulumirpc.EngineClient
|
|
|
|
|
streamID int32
|
|
|
|
|
severity pulumirpc.LogSeverity
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (w *logWriter) Write(p []byte) (n int, err error) {
|
|
|
|
|
val := string(p)
|
|
|
|
|
if _, err := w.engineClient.Log(w.ctx, &pulumirpc.LogRequest{
|
|
|
|
|
Message: strings.ToValidUTF8(val, "<22>"),
|
|
|
|
|
Urn: "",
|
|
|
|
|
Ephemeral: true,
|
|
|
|
|
StreamId: w.streamID,
|
|
|
|
|
Severity: w.severity,
|
|
|
|
|
}); err != nil {
|
|
|
|
|
return 0, err
|
|
|
|
|
}
|
|
|
|
|
return len(val), nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// These packages are known not to have any plugins.
|
2021-12-15 18:41:44 +00:00
|
|
|
|
// TODO[pulumi/pulumi#5863]: Remove this once the `pulumi-policy` package includes a `pulumi-plugin.json`
|
2020-12-04 03:22:16 +00:00
|
|
|
|
// file that indicates the package does not have an associated plugin, and enough time has passed.
|
|
|
|
|
var packagesWithoutPlugins = map[string]struct{}{
|
|
|
|
|
"pulumi-policy": {},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type pythonPackage struct {
|
2022-10-09 14:58:33 +00:00
|
|
|
|
Name string `json:"name"`
|
|
|
|
|
Version string `json:"version"`
|
|
|
|
|
Location string `json:"location"`
|
|
|
|
|
plugin *plugin.PulumiPluginJSON
|
2021-12-02 00:07:49 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Returns if pkg is a pulumi package.
|
|
|
|
|
//
|
|
|
|
|
// We check:
|
2021-12-15 18:41:44 +00:00
|
|
|
|
// 1. If there is a pulumi-plugin.json file.
|
2021-12-02 00:07:49 +00:00
|
|
|
|
// 2. If the first segment is "pulumi". This implies a first party package.
|
|
|
|
|
func (pkg *pythonPackage) isPulumiPackage() bool {
|
|
|
|
|
plugin, err := pkg.readPulumiPluginJSON()
|
|
|
|
|
if err == nil && plugin != nil {
|
|
|
|
|
return true
|
|
|
|
|
}
|
|
|
|
|
|
2024-03-04 07:23:34 +00:00
|
|
|
|
return strings.HasPrefix(pkg.Name, "pulumi_") || strings.HasPrefix(pkg.Name, "pulumi-")
|
2021-12-02 00:07:49 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (pkg *pythonPackage) readPulumiPluginJSON() (*plugin.PulumiPluginJSON, error) {
|
|
|
|
|
if pkg.plugin != nil {
|
|
|
|
|
return pkg.plugin, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// The name of the module inside the package can be different from the package name.
|
|
|
|
|
// However, our convention is to always use the same name, e.g. a package name of
|
|
|
|
|
// "pulumi-aws" will have a module named "pulumi_aws", so we can determine the module
|
|
|
|
|
// by replacing hyphens with underscores.
|
|
|
|
|
packageModuleName := strings.ReplaceAll(pkg.Name, "-", "_")
|
2021-12-15 18:41:44 +00:00
|
|
|
|
pulumiPluginFilePath := filepath.Join(pkg.Location, packageModuleName, "pulumi-plugin.json")
|
|
|
|
|
logging.V(5).Infof("readPulumiPluginJSON: pulumi-plugin.json file path: %s", pulumiPluginFilePath)
|
2021-12-02 00:07:49 +00:00
|
|
|
|
|
|
|
|
|
plugin, err := plugin.LoadPulumiPluginJSON(pulumiPluginFilePath)
|
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
|
return nil, nil
|
|
|
|
|
} else if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
pkg.plugin = plugin
|
|
|
|
|
return plugin, nil
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
2022-06-09 21:57:56 +00:00
|
|
|
|
func determinePulumiPackages(ctx context.Context, virtualenv, cwd string) ([]pythonPackage, error) {
|
2020-12-04 03:22:16 +00:00
|
|
|
|
logging.V(5).Infof("GetRequiredPlugins: Determining pulumi packages")
|
|
|
|
|
|
2021-08-25 15:40:58 +00:00
|
|
|
|
// Run the `python -m pip list -v --format json` command.
|
|
|
|
|
args := []string{"-m", "pip", "list", "-v", "--format", "json"}
|
2022-06-09 21:57:56 +00:00
|
|
|
|
output, err := runPythonCommand(ctx, virtualenv, cwd, args...)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
if err != nil {
|
2023-01-03 10:19:35 +00:00
|
|
|
|
return nil, fmt.Errorf("calling `python %s`: %w", strings.Join(args, " "), err)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
2021-08-25 15:40:58 +00:00
|
|
|
|
// Parse the JSON output; on some systems pip -v verbose mode
|
|
|
|
|
// follows JSON with non-JSON trailer, so we need to be
|
|
|
|
|
// careful when parsing and ignore the trailer.
|
2020-12-04 03:22:16 +00:00
|
|
|
|
var packages []pythonPackage
|
2021-08-25 15:40:58 +00:00
|
|
|
|
jsonDecoder := json.NewDecoder(bytes.NewBuffer(output))
|
|
|
|
|
if err := jsonDecoder.Decode(&packages); err != nil {
|
2023-01-03 10:19:35 +00:00
|
|
|
|
return nil, fmt.Errorf("parsing `python %s` output: %w", strings.Join(args, " "), err)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Only return Pulumi packages.
|
2023-06-28 16:02:04 +00:00
|
|
|
|
pulumiPackages := slice.Prealloc[pythonPackage](len(packages))
|
2020-12-04 03:22:16 +00:00
|
|
|
|
for _, pkg := range packages {
|
2021-12-02 00:07:49 +00:00
|
|
|
|
if !pkg.isPulumiPackage() {
|
2020-12-04 03:22:16 +00:00
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Skip packages that are known not have an associated plugin.
|
|
|
|
|
if _, ok := packagesWithoutPlugins[pkg.Name]; ok {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pulumiPackages = append(pulumiPackages, pkg)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
logging.V(5).Infof("GetRequiredPlugins: Pulumi packages: %#v", pulumiPackages)
|
|
|
|
|
|
|
|
|
|
return pulumiPackages, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// determinePluginDependency attempts to determine a plugin associated with a package. It checks to see if the package
|
2021-12-15 18:41:44 +00:00
|
|
|
|
// contains a pulumi-plugin.json file and uses the information in that file to determine the plugin. If `resource` in
|
|
|
|
|
// pulumi-plugin.json is set to false, nil is returned. If the name or version aren't specified in the file, these
|
|
|
|
|
// values are derived from the package name and version. If the plugin version cannot be determined from the package
|
|
|
|
|
// version, nil is returned.
|
2020-12-04 03:22:16 +00:00
|
|
|
|
func determinePluginDependency(
|
2023-03-03 16:36:39 +00:00
|
|
|
|
virtualenv, cwd string, pkg pythonPackage,
|
|
|
|
|
) (*pulumirpc.PluginDependency, error) {
|
2020-12-04 03:22:16 +00:00
|
|
|
|
var name, version, server string
|
2021-12-02 00:07:49 +00:00
|
|
|
|
plugin, err := pkg.readPulumiPluginJSON()
|
|
|
|
|
if plugin != nil && err == nil {
|
2020-12-04 03:22:16 +00:00
|
|
|
|
// If `resource` is set to false, the Pulumi package has indicated that there is no associated plugin.
|
|
|
|
|
// Ignore it.
|
|
|
|
|
if !plugin.Resource {
|
2021-08-25 15:40:58 +00:00
|
|
|
|
logging.V(5).Infof("GetRequiredPlugins: Ignoring package %s with resource set to false", pkg.Name)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
return nil, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
name, version, server = plugin.Name, plugin.Version, plugin.Server
|
2021-12-02 00:07:49 +00:00
|
|
|
|
} else if err != nil {
|
2020-12-04 03:22:16 +00:00
|
|
|
|
logging.V(5).Infof("GetRequiredPlugins: err: %v", err)
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if name == "" {
|
2021-08-25 15:40:58 +00:00
|
|
|
|
name = strings.TrimPrefix(pkg.Name, "pulumi-")
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if version == "" {
|
|
|
|
|
// The packageVersion may include additional pre-release tags (e.g. "2.14.0a1605583329" for an alpha
|
|
|
|
|
// release, "2.14.0b1605583329" for a beta release, "2.14.0rc1605583329" for an rc release, etc.).
|
|
|
|
|
// Unfortunately, this is not enough information to determine the plugin version. A package version of
|
|
|
|
|
// "3.31.0a1605189729" will have an associated plugin with a version of "3.31.0-alpha.1605189729+42435656".
|
|
|
|
|
// The "+42435656" suffix cannot be determined so the plugin version cannot be determined. In such cases,
|
|
|
|
|
// log the issue and skip the package.
|
2021-08-25 15:40:58 +00:00
|
|
|
|
version, err = determinePluginVersion(pkg.Version)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
if err != nil {
|
|
|
|
|
logging.V(5).Infof(
|
2022-02-10 10:25:53 +00:00
|
|
|
|
"GetRequiredPlugins: Could not determine plugin version for package %s with version %s: %s",
|
2023-12-20 15:54:06 +00:00
|
|
|
|
pkg.Name, pkg.Version, err)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
return nil, nil
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if !strings.HasPrefix(version, "v") {
|
|
|
|
|
// Add "v" prefix if not already present.
|
|
|
|
|
version = fmt.Sprintf("v%s", version)
|
|
|
|
|
}
|
|
|
|
|
|
2022-06-11 07:52:24 +00:00
|
|
|
|
result := &pulumirpc.PluginDependency{
|
2020-12-04 03:22:16 +00:00
|
|
|
|
Name: name,
|
|
|
|
|
Version: version,
|
|
|
|
|
Kind: "resource",
|
|
|
|
|
Server: server,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
logging.V(5).Infof("GetRequiredPlugins: Determining plugin dependency: %#v", result)
|
2022-06-11 07:52:24 +00:00
|
|
|
|
return result, nil
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// determinePluginVersion attempts to convert a PEP440 package version into a plugin version.
|
2022-02-10 10:25:53 +00:00
|
|
|
|
//
|
|
|
|
|
// Supported versions:
|
2022-08-16 13:25:52 +00:00
|
|
|
|
//
|
|
|
|
|
// PEP440 defines a version as `[N!]N(.N)*[{a|b|rc}N][.postN][.devN]`, but
|
|
|
|
|
// determinePluginVersion only supports a subset of that. Translations are provided for
|
|
|
|
|
// `N(.N)*[{a|b|rc}N][.postN][.devN]`.
|
2022-02-10 10:25:53 +00:00
|
|
|
|
//
|
|
|
|
|
// Translations:
|
|
|
|
|
//
|
2022-08-16 13:25:52 +00:00
|
|
|
|
// We ensure that there are at least 3 version segments. Missing segments are `0`
|
|
|
|
|
// padded.
|
|
|
|
|
// Example: 1.0 => 1.0.0
|
|
|
|
|
//
|
|
|
|
|
// We translate a,b,rc to alpha,beta,rc respectively with a hyphen separator.
|
|
|
|
|
// Example: 1.2.3a4 => 1.2.3-alpha.4, 1.2.3rc4 => 1.2.3-rc.4
|
2022-02-10 10:25:53 +00:00
|
|
|
|
//
|
2022-08-16 13:25:52 +00:00
|
|
|
|
// We translate `.post` and `.dev` by replacing the `.` with a `+`. If both `.post`
|
|
|
|
|
// and `.dev` are present, only one separator is used.
|
|
|
|
|
// Example: 1.2.3.post4 => 1.2.3+post4, 1.2.3.post4.dev5 => 1.2.3+post4dev5
|
2022-02-10 10:25:53 +00:00
|
|
|
|
//
|
|
|
|
|
// Reference on PEP440: https://www.python.org/dev/peps/pep-0440/
|
2020-12-04 03:22:16 +00:00
|
|
|
|
func determinePluginVersion(packageVersion string) (string, error) {
|
2022-02-10 10:25:53 +00:00
|
|
|
|
if len(packageVersion) == 0 {
|
2022-10-09 14:58:33 +00:00
|
|
|
|
return "", fmt.Errorf("cannot parse empty string")
|
2022-02-10 10:25:53 +00:00
|
|
|
|
}
|
|
|
|
|
// Verify ASCII
|
|
|
|
|
for i := 0; i < len(packageVersion); i++ {
|
|
|
|
|
c := packageVersion[i]
|
|
|
|
|
if c > unicode.MaxASCII {
|
|
|
|
|
return "", fmt.Errorf("byte %d is not ascii", i)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
parseNumber := func(s string) (string, string) {
|
|
|
|
|
i := 0
|
gosimple: Simplify loops
This replaces for loops and slice appends reported by gosimple
with simpler variants.
Specifically,
for _, x := range src {
dst = append(dst, x)
}
// can be replaced with
dst = append(dst, src...)
And,
for i, x := range src {
dst[i] = x
}
// can be replaced with
copy(dst, src)
And,
for true { ... }
// can be replaced with
for { ... }
And, given a string `s`,
for _, r := range []rune(s) { .. }
// can be replaced with
for _, r := range s { .. }
Lastly, this fixes in ineffective break statement
also reported by the linter.
Inside a switch block,
`break` affects the current `case` only.
The outer loop needs a label.
2023-01-11 15:58:17 +00:00
|
|
|
|
for _, c := range s {
|
2022-02-10 10:25:53 +00:00
|
|
|
|
if c > '9' || c < '0' {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
i++
|
|
|
|
|
}
|
|
|
|
|
return s[:i], s[i:]
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Explicitly err on epochs
|
|
|
|
|
if num, maybeEpoch := parseNumber(packageVersion); num != "" && strings.HasPrefix(maybeEpoch, "!") {
|
2022-10-09 14:58:33 +00:00
|
|
|
|
return "", fmt.Errorf("epochs are not supported")
|
2022-02-10 10:25:53 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
segments := []string{}
|
|
|
|
|
num, rest := "", packageVersion
|
|
|
|
|
foundDot := false
|
|
|
|
|
for {
|
|
|
|
|
if num, rest = parseNumber(rest); num != "" {
|
|
|
|
|
foundDot = false
|
|
|
|
|
segments = append(segments, num)
|
|
|
|
|
if strings.HasPrefix(rest, ".") {
|
|
|
|
|
rest = rest[1:]
|
|
|
|
|
foundDot = true
|
|
|
|
|
} else {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if foundDot {
|
|
|
|
|
rest = "." + rest
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for len(segments) < 3 {
|
|
|
|
|
segments = append(segments, "0")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if rest == "" {
|
|
|
|
|
r := strings.Join(segments, ".")
|
|
|
|
|
return r, nil
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
2022-02-10 10:25:53 +00:00
|
|
|
|
var preRelease string
|
|
|
|
|
|
|
|
|
|
switch {
|
|
|
|
|
case rest[0] == 'a':
|
|
|
|
|
preRelease, rest = parseNumber(rest[1:])
|
|
|
|
|
preRelease = "-alpha." + preRelease
|
|
|
|
|
case rest[0] == 'b':
|
|
|
|
|
preRelease, rest = parseNumber(rest[1:])
|
|
|
|
|
preRelease = "-beta." + preRelease
|
|
|
|
|
case strings.HasPrefix(rest, "rc"):
|
|
|
|
|
preRelease, rest = parseNumber(rest[2:])
|
|
|
|
|
preRelease = "-rc." + preRelease
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var postRelease string
|
|
|
|
|
if strings.HasPrefix(rest, ".post") {
|
|
|
|
|
postRelease, rest = parseNumber(rest[5:])
|
|
|
|
|
postRelease = "+post" + postRelease
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var developmentRelease string
|
|
|
|
|
if strings.HasPrefix(rest, ".dev") {
|
|
|
|
|
developmentRelease, rest = parseNumber(rest[4:])
|
|
|
|
|
join := ""
|
|
|
|
|
if postRelease == "" {
|
|
|
|
|
join = "+"
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
2022-02-10 10:25:53 +00:00
|
|
|
|
developmentRelease = join + "dev" + developmentRelease
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
2022-02-10 10:25:53 +00:00
|
|
|
|
if rest != "" {
|
|
|
|
|
return "", fmt.Errorf("'%s' still unparsed", rest)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
result := strings.Join(segments, ".") + preRelease + postRelease + developmentRelease
|
|
|
|
|
|
|
|
|
|
return result, nil
|
2020-12-04 03:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
2022-06-09 21:57:56 +00:00
|
|
|
|
func runPythonCommand(ctx context.Context, virtualenv, cwd string, arg ...string) ([]byte, error) {
|
2020-12-04 03:22:16 +00:00
|
|
|
|
var err error
|
|
|
|
|
var cmd *exec.Cmd
|
|
|
|
|
if virtualenv != "" {
|
2022-07-15 15:55:52 +00:00
|
|
|
|
// Default to the "python" executable in the virtual environment, but allow the user to override it
|
|
|
|
|
// with PULUMI_PYTHON_CMD.
|
|
|
|
|
pythonCmd := os.Getenv("PULUMI_PYTHON_CMD")
|
|
|
|
|
if pythonCmd == "" {
|
|
|
|
|
pythonCmd = "python"
|
|
|
|
|
}
|
|
|
|
|
cmd = python.VirtualEnvCommand(virtualenv, pythonCmd, arg...)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
} else {
|
2022-06-09 21:57:56 +00:00
|
|
|
|
cmd, err = python.Command(ctx, arg...)
|
2020-12-04 03:22:16 +00:00
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if logging.V(5) {
|
|
|
|
|
commandStr := strings.Join(arg, " ")
|
|
|
|
|
logging.V(5).Infof("Language host launching process: %s %s", cmd.Path, commandStr)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
cmd.Dir = cwd
|
|
|
|
|
output, err := cmd.Output()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if logging.V(9) {
|
|
|
|
|
logging.V(9).Infof("Process output: %s", string(output))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return output, err
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
}
|
|
|
|
|
|
2022-10-09 14:58:33 +00:00
|
|
|
|
// Run is RPC endpoint for LanguageRuntimeServer::Run
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
func (host *pythonLanguageHost) Run(ctx context.Context, req *pulumirpc.RunRequest) (*pulumirpc.RunResponse, error) {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
opts, err := parseOptions(req.Info.RootDirectory, req.Info.Options.AsMap())
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-15 05:16:43 +00:00
|
|
|
|
args := []string{host.exec}
|
|
|
|
|
args = append(args, host.constructArguments(req)...)
|
|
|
|
|
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
config, err := host.constructConfig(req)
|
|
|
|
|
if err != nil {
|
2023-01-03 10:19:35 +00:00
|
|
|
|
err = fmt.Errorf("failed to serialize configuration: %w", err)
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
return nil, err
|
|
|
|
|
}
|
2021-05-18 22:00:30 +00:00
|
|
|
|
configSecretKeys, err := host.constructConfigSecretKeys(req)
|
|
|
|
|
if err != nil {
|
2023-01-03 10:19:35 +00:00
|
|
|
|
err = fmt.Errorf("failed to serialize configuration secret keys: %w", err)
|
2021-05-18 22:00:30 +00:00
|
|
|
|
return nil, err
|
|
|
|
|
}
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
|
2018-05-15 22:28:00 +00:00
|
|
|
|
if logging.V(5) {
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
commandStr := strings.Join(args, " ")
|
2018-05-15 22:28:00 +00:00
|
|
|
|
logging.V(5).Infoln("Language host launching process: ", host.exec, commandStr)
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Now simply spawn a process to execute the requested program, wiring up stdout/stderr directly.
|
|
|
|
|
var errResult string
|
2020-06-09 23:42:53 +00:00
|
|
|
|
var cmd *exec.Cmd
|
|
|
|
|
var virtualenv string
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
if opts.virtualenv != "" {
|
|
|
|
|
virtualenv = opts.virtualenvPath
|
2020-06-09 23:42:53 +00:00
|
|
|
|
if !python.IsVirtualEnv(virtualenv) {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
return nil, python.NewVirtualEnvError(opts.virtualenv, virtualenv)
|
2020-06-09 23:42:53 +00:00
|
|
|
|
}
|
|
|
|
|
cmd = python.VirtualEnvCommand(virtualenv, "python", args...)
|
|
|
|
|
} else {
|
2022-06-09 21:57:56 +00:00
|
|
|
|
cmd, err = python.Command(ctx, args...)
|
2020-06-09 23:42:53 +00:00
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
2018-11-19 22:54:24 +00:00
|
|
|
|
}
|
|
|
|
|
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
cmd.Stdout = os.Stdout
|
|
|
|
|
cmd.Stderr = os.Stderr
|
2021-05-18 22:00:30 +00:00
|
|
|
|
if virtualenv != "" || config != "" || configSecretKeys != "" {
|
2020-06-09 23:42:53 +00:00
|
|
|
|
env := os.Environ()
|
|
|
|
|
if virtualenv != "" {
|
|
|
|
|
env = python.ActivateVirtualEnv(env, virtualenv)
|
|
|
|
|
}
|
|
|
|
|
if config != "" {
|
|
|
|
|
env = append(env, pulumiConfigVar+"="+config)
|
|
|
|
|
}
|
2021-05-18 22:00:30 +00:00
|
|
|
|
if configSecretKeys != "" {
|
|
|
|
|
env = append(env, pulumiConfigSecretKeysVar+"="+configSecretKeys)
|
|
|
|
|
}
|
2020-06-09 23:42:53 +00:00
|
|
|
|
cmd.Env = env
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
}
|
|
|
|
|
if err := cmd.Run(); err != nil {
|
2018-11-19 22:59:01 +00:00
|
|
|
|
// Python does not explicitly flush standard out or standard error when exiting abnormally. For this reason, we
|
|
|
|
|
// need to explicitly flush our output streams so that, when we exit, the engine picks up the child Python
|
|
|
|
|
// process's stdout and stderr writes.
|
|
|
|
|
//
|
|
|
|
|
// This is especially crucial for Python because it is possible for the child Python process to crash very fast
|
|
|
|
|
// if Pulumi is misconfigured, so we must be sure to present a high-quality error message to the user.
|
|
|
|
|
contract.IgnoreError(os.Stdout.Sync())
|
|
|
|
|
contract.IgnoreError(os.Stderr.Sync())
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
if exiterr, ok := err.(*exec.ExitError); ok {
|
|
|
|
|
// If the program ran, but exited with a non-zero error code. This will happen often, since user
|
|
|
|
|
// errors will trigger this. So, the error message should look as nice as possible.
|
|
|
|
|
if status, stok := exiterr.Sys().(syscall.WaitStatus); stok {
|
2022-08-11 17:38:34 +00:00
|
|
|
|
switch status.ExitStatus() {
|
|
|
|
|
case 0:
|
|
|
|
|
// This really shouldn't happen, but if it does, we don't want to render "non-zero exit code"
|
2023-01-03 10:19:35 +00:00
|
|
|
|
err = fmt.Errorf("program exited unexpectedly: %w", exiterr)
|
2022-08-11 17:38:34 +00:00
|
|
|
|
case pythonProcessExitedAfterShowingUserActionableMessage:
|
|
|
|
|
return &pulumirpc.RunResponse{Error: "", Bail: true}, nil
|
|
|
|
|
default:
|
2023-01-03 10:19:35 +00:00
|
|
|
|
err = fmt.Errorf("program exited with non-zero exit code: %d", status.ExitStatus())
|
2022-08-11 17:38:34 +00:00
|
|
|
|
}
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
} else {
|
2023-01-03 10:19:35 +00:00
|
|
|
|
err = fmt.Errorf("program exited unexpectedly: %w", exiterr)
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
// Otherwise, we didn't even get to run the program. This ought to never happen unless there's
|
|
|
|
|
// a bug or system condition that prevented us from running the language exec. Issue a scarier error.
|
2023-01-03 10:19:35 +00:00
|
|
|
|
err = fmt.Errorf("problem executing program (could not run language executor): %w", err)
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
errResult = err.Error()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return &pulumirpc.RunResponse{Error: errResult}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// constructArguments constructs a command-line for `pulumi-language-python`
|
|
|
|
|
// by enumerating all of the optional and non-optional arguments present
|
|
|
|
|
// in a RunRequest.
|
|
|
|
|
func (host *pythonLanguageHost) constructArguments(req *pulumirpc.RunRequest) []string {
|
|
|
|
|
var args []string
|
|
|
|
|
maybeAppendArg := func(k, v string) {
|
|
|
|
|
if v != "" {
|
|
|
|
|
args = append(args, "--"+k, v)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
maybeAppendArg("monitor", req.GetMonitorAddress())
|
|
|
|
|
maybeAppendArg("engine", host.engineAddress)
|
|
|
|
|
maybeAppendArg("project", req.GetProject())
|
|
|
|
|
maybeAppendArg("stack", req.GetStack())
|
|
|
|
|
maybeAppendArg("pwd", req.GetPwd())
|
2023-12-12 12:19:42 +00:00
|
|
|
|
maybeAppendArg("dry_run", strconv.FormatBool(req.GetDryRun()))
|
|
|
|
|
maybeAppendArg("parallel", strconv.Itoa(int(req.GetParallel())))
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
maybeAppendArg("tracing", host.tracing)
|
2022-08-31 09:33:29 +00:00
|
|
|
|
maybeAppendArg("organization", req.GetOrganization())
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
// The engine should always pass a name for entry point, even if its just "." for the program directory.
|
|
|
|
|
args = append(args, req.Info.EntryPoint)
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
|
|
|
|
|
args = append(args, req.GetArgs()...)
|
|
|
|
|
return args
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// constructConfig json-serializes the configuration data given as part of a RunRequest.
|
|
|
|
|
func (host *pythonLanguageHost) constructConfig(req *pulumirpc.RunRequest) (string, error) {
|
|
|
|
|
configMap := req.GetConfig()
|
|
|
|
|
if configMap == nil {
|
|
|
|
|
return "", nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
configJSON, err := json.Marshal(configMap)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return "", err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return string(configJSON), nil
|
|
|
|
|
}
|
|
|
|
|
|
2021-05-18 22:00:30 +00:00
|
|
|
|
// constructConfigSecretKeys JSON-serializes the list of keys that contain secret values given as part of
|
|
|
|
|
// a RunRequest.
|
|
|
|
|
func (host *pythonLanguageHost) constructConfigSecretKeys(req *pulumirpc.RunRequest) (string, error) {
|
|
|
|
|
configSecretKeys := req.GetConfigSecretKeys()
|
|
|
|
|
if configSecretKeys == nil {
|
|
|
|
|
return "[]", nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
configSecretKeysJSON, err := json.Marshal(configSecretKeys)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return "", err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return string(configSecretKeysJSON), nil
|
|
|
|
|
}
|
|
|
|
|
|
2024-01-17 09:35:20 +00:00
|
|
|
|
func (host *pythonLanguageHost) GetPluginInfo(ctx context.Context, req *emptypb.Empty) (*pulumirpc.PluginInfo, error) {
|
Get the empty Python program working
This change gets enough of the Python SDK up and running that the
empty Python program will work. Mostly just scaffolding, but the
basic structure is now in place. The primary remaining work is to
wire up resource creation to the gRPC interfaces.
In summary:
* The basic structure is as follows:
- Everything goes into sdk/python/.
- sdk/python/cmd/pulumi-langhost-python is a Go language host
that simply knows how to spawn Python processes to run out
entrypoint in response to requests by the engine.
- sdk/python/cmd/pulumi-langhost-python-exec is a little Python
shim that is invoked by the language host to run Python programs,
and is responsible for setting up the minimal goo before we can
do so (RPC connections and the like).
- sdk/python/lib/ contains a Python Pip package suitable for PyPi.
- In there, we have two packages: the root pulumi package that
contains all of the basic Pulumi programming model abstractions,
and pulumi.runtime, which contains the implementation of
resource registration, RPC interfacing with the engine, and so on.
* Add logic in our test framework to conditionalize on the language
type and react accordingly. This will allow us to skip Yarn for
Python projects and eventually run Pip if there's a requirements.txt.
* Created the basic project structure, including all of the usual
Make targets for installing into the proper places.
* Building also runs Pylint and we are clean.
There are a few other minor things in here:
* Add an "empty" test for both Node.js and Python. These pass.
* Fix an existing bug in plugin shutdown logic. At some point, we
started waiting for stderr/stdout to flush before shutting down
the plugin; but if certain failures happen "early" during the
plugin launch process, these channels will never get initialized
and so waiting for them deadlocks.
* Recently we seem to have added logic to delete test temp
directories if a failure happened during initialization of said
temp directories. This is unfortunate, because you often need to
look at the temp directory to see what failed. We already clean
them up elsewhere after the full test completes successfully, so
I don't think we need to be doing this, and I've removed it.
Still many loose ends (config, resources, etc), but it's a start!
2018-01-13 18:29:34 +00:00
|
|
|
|
return &pulumirpc.PluginInfo{
|
|
|
|
|
Version: version.Version,
|
|
|
|
|
}, nil
|
|
|
|
|
}
|
2021-11-09 22:22:16 +00:00
|
|
|
|
|
|
|
|
|
// validateVersion checks that python is running a valid version. If a version
|
|
|
|
|
// is invalid, it prints to os.Stderr. This is interpreted as diagnostic message
|
|
|
|
|
// by the Pulumi CLI program.
|
2022-06-09 21:57:56 +00:00
|
|
|
|
func validateVersion(ctx context.Context, virtualEnvPath string) {
|
2021-11-09 22:22:16 +00:00
|
|
|
|
var versionCmd *exec.Cmd
|
|
|
|
|
var err error
|
|
|
|
|
versionArgs := []string{"--version"}
|
2021-11-19 23:21:13 +00:00
|
|
|
|
if virtualEnvPath != "" {
|
2021-11-09 22:22:16 +00:00
|
|
|
|
versionCmd = python.VirtualEnvCommand(virtualEnvPath, "python", versionArgs...)
|
2022-06-09 21:57:56 +00:00
|
|
|
|
} else if versionCmd, err = python.Command(ctx, versionArgs...); err != nil {
|
2021-11-09 22:22:16 +00:00
|
|
|
|
fmt.Fprintf(os.Stderr, "Failed to find python executable\n")
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
var out []byte
|
|
|
|
|
if out, err = versionCmd.Output(); err != nil {
|
2023-12-20 15:54:06 +00:00
|
|
|
|
fmt.Fprintf(os.Stderr, "Failed to resolve python version command: %s\n", err)
|
2021-11-09 22:22:16 +00:00
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
version := strings.TrimSpace(strings.TrimPrefix(string(out), "Python "))
|
|
|
|
|
parsed, err := semver.Parse(version)
|
|
|
|
|
if err != nil {
|
|
|
|
|
fmt.Fprintf(os.Stderr, "Failed to parse python version: '%s'\n", version)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
if parsed.LT(minimumSupportedPythonVersion) {
|
|
|
|
|
fmt.Fprintf(os.Stderr, "Pulumi does not support Python %s."+
|
|
|
|
|
" Please upgrade to at least %s\n", parsed, minimumSupportedPythonVersion)
|
|
|
|
|
} else if parsed.LT(eolPythonVersion) {
|
|
|
|
|
fmt.Fprintf(os.Stderr, "Python %d.%d is approaching EOL and will not be supported in Pulumi soon."+
|
|
|
|
|
" Check %s for more details\n", parsed.Major,
|
2021-11-19 23:21:13 +00:00
|
|
|
|
parsed.Minor, eolPythonVersionIssue)
|
2021-11-09 22:22:16 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
2022-04-03 14:54:59 +00:00
|
|
|
|
|
|
|
|
|
func (host *pythonLanguageHost) InstallDependencies(
|
2023-03-03 16:36:39 +00:00
|
|
|
|
req *pulumirpc.InstallDependenciesRequest, server pulumirpc.LanguageRuntime_InstallDependenciesServer,
|
|
|
|
|
) error {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
opts, err := parseOptions(req.Info.RootDirectory, req.Info.Options.AsMap())
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
|
2022-10-04 08:58:01 +00:00
|
|
|
|
closer, stdout, stderr, err := rpcutil.MakeInstallDependenciesStreams(server, req.IsTerminal)
|
2022-04-03 14:54:59 +00:00
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
// best effort close, but we try an explicit close and error check at the end as well
|
|
|
|
|
defer closer.Close()
|
|
|
|
|
|
|
|
|
|
stdout.Write([]byte("Installing dependencies...\n\n"))
|
|
|
|
|
|
2022-06-09 21:57:56 +00:00
|
|
|
|
if err := python.InstallDependenciesWithWriters(server.Context(),
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
req.Info.ProgramDirectory, opts.virtualenvPath, true /*showOutput*/, stdout, stderr); err != nil {
|
2022-04-03 14:54:59 +00:00
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
stdout.Write([]byte("Finished installing dependencies\n\n"))
|
|
|
|
|
|
all: Fix revive issues
Fixes the following issues found by revive
included in the latest release of golangci-lint.
Full list of issues:
**pkg**
```
backend/display/object_diff.go:47:10: superfluous-else: if block ends with a break statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
backend/display/object_diff.go:716:12: redefines-builtin-id: redefinition of the built-in function delete (revive)
backend/display/object_diff.go:742:14: redefines-builtin-id: redefinition of the built-in function delete (revive)
backend/display/object_diff.go:983:10: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (revive)
backend/httpstate/backend.go:1814:4: redefines-builtin-id: redefinition of the built-in function cap (revive)
backend/httpstate/backend.go:1824:5: redefines-builtin-id: redefinition of the built-in function cap (revive)
backend/httpstate/client/client.go:444:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
backend/httpstate/client/client.go:455:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
cmd/pulumi/org.go:113:4: if-return: redundant if ...; err != nil check, just return error instead. (revive)
cmd/pulumi/util.go:216:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
codegen/docs/gen.go:428:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
codegen/hcl2/model/expression.go:2151:5: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:151:2: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:329:3: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:381:5: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/nodejs/gen.go:1367:5: redefines-builtin-id: redefinition of the built-in function copy (revive)
codegen/python/gen_program_expressions.go:136:2: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/python/gen_program_expressions.go:142:3: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/report/report.go:126:6: redefines-builtin-id: redefinition of the built-in function panic (revive)
codegen/schema/docs_test.go:210:10: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
codegen/schema/schema.go:790:2: redefines-builtin-id: redefinition of the built-in type any (revive)
codegen/schema/schema.go:793:4: redefines-builtin-id: redefinition of the built-in type any (revive)
resource/deploy/plan.go:506:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
resource/deploy/snapshot_test.go:59:3: redefines-builtin-id: redefinition of the built-in function copy (revive)
resource/deploy/state_builder.go:108:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
```
**sdk**
```
go/common/resource/plugin/context.go:142:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
go/common/resource/plugin/plugin.go:142:12: superfluous-else: if block ends with a break statement, so drop this else and outdent its block (revive)
go/common/resource/properties_diff.go:114:2: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:117:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:122:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:127:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:132:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/util/deepcopy/copy.go:30:1: redefines-builtin-id: redefinition of the built-in function copy (revive)
go/common/workspace/creds.go:242:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi-language-go/main.go:569:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi-language-go/main.go:706:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi/run_test.go:925:2: redefines-builtin-id: redefinition of the built-in type any (revive)
go/pulumi/run_test.go:933:3: redefines-builtin-id: redefinition of the built-in type any (revive)
nodejs/cmd/pulumi-language-nodejs/main.go:778:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/cmd/pulumi-language-python/main.go:1011:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/cmd/pulumi-language-python/main.go:863:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/python.go:230:2: redefines-builtin-id: redefinition of the built-in function print (revive)
```
**tests**
```
integration/integration_util_test.go:282:11: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
```
2023-03-20 23:48:02 +00:00
|
|
|
|
return closer.Close()
|
2022-04-03 14:54:59 +00:00
|
|
|
|
}
|
2022-07-25 11:35:16 +00:00
|
|
|
|
|
2024-01-17 09:35:20 +00:00
|
|
|
|
func (host *pythonLanguageHost) About(ctx context.Context, req *emptypb.Empty) (*pulumirpc.AboutResponse, error) {
|
2022-08-15 13:55:04 +00:00
|
|
|
|
errCouldNotGet := func(err error) (*pulumirpc.AboutResponse, error) {
|
|
|
|
|
return nil, fmt.Errorf("failed to get version: %w", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var cmd *exec.Cmd
|
|
|
|
|
// if CommandPath has an error, then so will Command. The error can
|
|
|
|
|
// therefore be ignored as redundant.
|
|
|
|
|
pyexe, _, _ := python.CommandPath()
|
|
|
|
|
cmd, err := python.Command(ctx, "--version")
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
var out []byte
|
|
|
|
|
if out, err = cmd.Output(); err != nil {
|
|
|
|
|
return errCouldNotGet(err)
|
|
|
|
|
}
|
2023-09-23 08:37:57 +00:00
|
|
|
|
version := strings.TrimSpace(strings.TrimPrefix(string(out), "Python "))
|
2022-08-15 13:55:04 +00:00
|
|
|
|
|
|
|
|
|
return &pulumirpc.AboutResponse{
|
|
|
|
|
Executable: pyexe,
|
|
|
|
|
Version: version,
|
|
|
|
|
}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Calls a python command as pulumi would. This means we need to accommodate for
|
|
|
|
|
// a virtual environment if it exists.
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
func (host *pythonLanguageHost) callPythonCommand(
|
|
|
|
|
ctx context.Context, virtualenvPath string, args ...string,
|
|
|
|
|
) (string, error) {
|
|
|
|
|
if virtualenvPath == "" {
|
2022-08-15 13:55:04 +00:00
|
|
|
|
return callPythonCommandNoEnvironment(ctx, args...)
|
|
|
|
|
}
|
|
|
|
|
// We now know that a virtual environment exists.
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
cmd := python.VirtualEnvCommand(virtualenvPath, "python", args...)
|
2022-08-15 13:55:04 +00:00
|
|
|
|
result, err := cmd.Output()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return "", err
|
|
|
|
|
}
|
|
|
|
|
return string(result), nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Call a python command in a runtime agnostic way. Call python from the path.
|
|
|
|
|
// Do not use a virtual environment.
|
|
|
|
|
func callPythonCommandNoEnvironment(ctx context.Context, args ...string) (string, error) {
|
|
|
|
|
cmd, err := python.Command(ctx, args...)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return "", err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var result []byte
|
|
|
|
|
if result, err = cmd.Output(); err != nil {
|
|
|
|
|
return "", err
|
|
|
|
|
}
|
|
|
|
|
return string(result), nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type pipDependency struct {
|
|
|
|
|
Name string `json:"name"`
|
|
|
|
|
Version string `json:"version"`
|
2022-07-25 11:35:16 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (host *pythonLanguageHost) GetProgramDependencies(
|
2023-03-03 16:36:39 +00:00
|
|
|
|
ctx context.Context, req *pulumirpc.GetProgramDependenciesRequest,
|
|
|
|
|
) (*pulumirpc.GetProgramDependenciesResponse, error) {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
opts, err := parseOptions(req.Info.RootDirectory, req.Info.Options.AsMap())
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
2022-08-15 13:55:04 +00:00
|
|
|
|
cmdArgs := []string{"-m", "pip", "list", "--format=json"}
|
|
|
|
|
if !req.TransitiveDependencies {
|
|
|
|
|
cmdArgs = append(cmdArgs, "--not-required")
|
|
|
|
|
}
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
out, err := host.callPythonCommand(ctx, opts.virtualenvPath, cmdArgs...)
|
2022-08-15 13:55:04 +00:00
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
var result []pipDependency
|
|
|
|
|
err = json.Unmarshal([]byte(out), &result)
|
|
|
|
|
if err != nil {
|
2022-10-09 14:58:33 +00:00
|
|
|
|
return nil, fmt.Errorf("failed to parse \"python %s\" result: %w", strings.Join(cmdArgs, " "), err)
|
2022-08-15 13:55:04 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
dependencies := make([]*pulumirpc.DependencyInfo, len(result))
|
|
|
|
|
for i, dep := range result {
|
|
|
|
|
dependencies[i] = &pulumirpc.DependencyInfo{
|
|
|
|
|
Name: dep.Name,
|
|
|
|
|
Version: dep.Version,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return &pulumirpc.GetProgramDependenciesResponse{
|
|
|
|
|
Dependencies: dependencies,
|
|
|
|
|
}, nil
|
2022-07-25 11:35:16 +00:00
|
|
|
|
}
|
2022-10-04 08:58:01 +00:00
|
|
|
|
|
|
|
|
|
func (host *pythonLanguageHost) RunPlugin(
|
2023-03-03 16:36:39 +00:00
|
|
|
|
req *pulumirpc.RunPluginRequest, server pulumirpc.LanguageRuntime_RunPluginServer,
|
|
|
|
|
) error {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
logging.V(5).Infof("Attempting to run python plugin in %s", req.Info.ProgramDirectory)
|
|
|
|
|
|
|
|
|
|
opts, err := parseOptions(req.Info.RootDirectory, req.Info.Options.AsMap())
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
2023-03-06 14:14:49 +00:00
|
|
|
|
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
args := []string{req.Info.ProgramDirectory}
|
2023-03-06 14:14:49 +00:00
|
|
|
|
args = append(args, req.Args...)
|
|
|
|
|
|
|
|
|
|
var cmd *exec.Cmd
|
|
|
|
|
var virtualenv string
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
if opts.virtualenv != "" {
|
|
|
|
|
virtualenv = opts.virtualenvPath
|
2023-03-06 14:14:49 +00:00
|
|
|
|
if !python.IsVirtualEnv(virtualenv) {
|
Pass root and main info to language host methods (#14654)
<!---
Thanks so much for your contribution! If this is your first time
contributing, please ensure that you have read the
[CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md)
documentation.
-->
# Description
<!--- Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context. -->
This is two changes rolled together in a way.
Firstly passing some of the data that we pass on language runtime
startup to also pass it to Run/GetRequiredPlugins/etc. This is needed
for matrix testing, as we only get to start the language runtime up once
for that but want to execute multiple programs with it.
I feel it's also a little more consistent as we use the language
runtimes in other contexts (codegen) where there isn't really a root
directory, and aren't any options (and if we did do options the options
for codegen are not going to be the same as for execution). It also
means we can reuse a language host for shimless and substack programs,
as before they heavily relied on their current working directory to
calculate paths, and obviosly could only take one set of options at
startup. Imagine a shimless python package + a python root program, that
would have needed two startups of the python language host to deal with,
this unblocks it so we can make the engine smarter and only use one.
Secondly renaming some of the fields we pass to
Run/GetRequiredPlugins/etc today. `Pwd` and `Program` were not very
descriptive and had pretty non-obvious documentation:
```
string pwd = 3; // the program's working directory.
string program = 4; // the path to the program to execute.
```
`pwd` will remain, although probably rename it to `working_directory` at
some point, because while today we always start programs up with the
working directory equal to the program directory that definitely is
going to change in the future (at least for MLCs and substack programs).
But the name `pwd` doesn't make it clear that this was intended to be
the working directory _and_ the directory which contains the program.
`program` was in fact nearly always ".", and if it wasn't that it was
just a filename. The engine never sent a path for `program` (although we
did have some unit tests to check how that worked for the nodejs and
python hosts).
These are now replaced by a new structure with (I think) more clearly
named and documented fields (see ProgramInfo in langauge.proto).
The engine still sends the old data for now, we need to update
dotnet/yaml/java before we break the old interface and give Virtus Labs
a chance to update [besom](https://github.com/VirtusLab/besom).
## Checklist
- [x] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
- [ ] I have formatted my code using `gofumpt`
<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!---
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @Pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->
2023-12-10 17:30:51 +00:00
|
|
|
|
return python.NewVirtualEnvError(opts.virtualenv, virtualenv)
|
2023-03-06 14:14:49 +00:00
|
|
|
|
}
|
|
|
|
|
cmd = python.VirtualEnvCommand(virtualenv, "python", args...)
|
|
|
|
|
} else {
|
|
|
|
|
var err error
|
|
|
|
|
cmd, err = python.Command(server.Context(), args...)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
closer, stdout, stderr, err := rpcutil.MakeRunPluginStreams(server, false)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
// best effort close, but we try an explicit close and error check at the end as well
|
|
|
|
|
defer closer.Close()
|
|
|
|
|
|
|
|
|
|
cmd.Dir = req.Pwd
|
|
|
|
|
cmd.Env = req.Env
|
|
|
|
|
cmd.Stdout, cmd.Stderr = stdout, stderr
|
|
|
|
|
|
|
|
|
|
if virtualenv != "" {
|
|
|
|
|
cmd.Env = python.ActivateVirtualEnv(cmd.Env, virtualenv)
|
|
|
|
|
}
|
|
|
|
|
if err = cmd.Run(); err != nil {
|
|
|
|
|
var exiterr *exec.ExitError
|
|
|
|
|
if errors.As(err, &exiterr) {
|
|
|
|
|
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
|
|
|
|
|
return server.Send(&pulumirpc.RunPluginResponse{
|
|
|
|
|
Output: &pulumirpc.RunPluginResponse_Exitcode{Exitcode: int32(status.ExitStatus())},
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
return fmt.Errorf("program exited unexpectedly: %w", exiterr)
|
|
|
|
|
}
|
|
|
|
|
return fmt.Errorf("problem executing plugin program (could not run language executor): %w", err)
|
|
|
|
|
}
|
|
|
|
|
|
all: Fix revive issues
Fixes the following issues found by revive
included in the latest release of golangci-lint.
Full list of issues:
**pkg**
```
backend/display/object_diff.go:47:10: superfluous-else: if block ends with a break statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
backend/display/object_diff.go:716:12: redefines-builtin-id: redefinition of the built-in function delete (revive)
backend/display/object_diff.go:742:14: redefines-builtin-id: redefinition of the built-in function delete (revive)
backend/display/object_diff.go:983:10: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (revive)
backend/httpstate/backend.go:1814:4: redefines-builtin-id: redefinition of the built-in function cap (revive)
backend/httpstate/backend.go:1824:5: redefines-builtin-id: redefinition of the built-in function cap (revive)
backend/httpstate/client/client.go:444:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
backend/httpstate/client/client.go:455:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
cmd/pulumi/org.go:113:4: if-return: redundant if ...; err != nil check, just return error instead. (revive)
cmd/pulumi/util.go:216:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
codegen/docs/gen.go:428:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
codegen/hcl2/model/expression.go:2151:5: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:151:2: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:329:3: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/hcl2/syntax/comments.go:381:5: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/nodejs/gen.go:1367:5: redefines-builtin-id: redefinition of the built-in function copy (revive)
codegen/python/gen_program_expressions.go:136:2: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/python/gen_program_expressions.go:142:3: redefines-builtin-id: redefinition of the built-in function close (revive)
codegen/report/report.go:126:6: redefines-builtin-id: redefinition of the built-in function panic (revive)
codegen/schema/docs_test.go:210:10: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
codegen/schema/schema.go:790:2: redefines-builtin-id: redefinition of the built-in type any (revive)
codegen/schema/schema.go:793:4: redefines-builtin-id: redefinition of the built-in type any (revive)
resource/deploy/plan.go:506:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
resource/deploy/snapshot_test.go:59:3: redefines-builtin-id: redefinition of the built-in function copy (revive)
resource/deploy/state_builder.go:108:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
```
**sdk**
```
go/common/resource/plugin/context.go:142:2: redefines-builtin-id: redefinition of the built-in function copy (revive)
go/common/resource/plugin/plugin.go:142:12: superfluous-else: if block ends with a break statement, so drop this else and outdent its block (revive)
go/common/resource/properties_diff.go:114:2: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:117:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:122:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:127:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/resource/properties_diff.go:132:4: redefines-builtin-id: redefinition of the built-in function len (revive)
go/common/util/deepcopy/copy.go:30:1: redefines-builtin-id: redefinition of the built-in function copy (revive)
go/common/workspace/creds.go:242:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi-language-go/main.go:569:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi-language-go/main.go:706:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
go/pulumi/run_test.go:925:2: redefines-builtin-id: redefinition of the built-in type any (revive)
go/pulumi/run_test.go:933:3: redefines-builtin-id: redefinition of the built-in type any (revive)
nodejs/cmd/pulumi-language-nodejs/main.go:778:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/cmd/pulumi-language-python/main.go:1011:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/cmd/pulumi-language-python/main.go:863:2: if-return: redundant if ...; err != nil check, just return error instead. (revive)
python/python.go:230:2: redefines-builtin-id: redefinition of the built-in function print (revive)
```
**tests**
```
integration/integration_util_test.go:282:11: superfluous-else: if block ends with a continue statement, so drop this else and outdent its block (move short variable declaration to its own line if necessary) (revive)
```
2023-03-20 23:48:02 +00:00
|
|
|
|
return closer.Close()
|
2022-10-04 08:58:01 +00:00
|
|
|
|
}
|
2023-08-31 16:35:21 +00:00
|
|
|
|
|
|
|
|
|
func (host *pythonLanguageHost) GenerateProject(
|
|
|
|
|
ctx context.Context, req *pulumirpc.GenerateProjectRequest,
|
|
|
|
|
) (*pulumirpc.GenerateProjectResponse, error) {
|
|
|
|
|
loader, err := schema.NewLoaderClient(req.LoaderTarget)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var extraOptions []pcl.BindOption
|
|
|
|
|
if !req.Strict {
|
|
|
|
|
extraOptions = append(extraOptions, pcl.NonStrictBindOptions()...)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// for python, prefer output-versioned invokes
|
|
|
|
|
extraOptions = append(extraOptions, pcl.PreferOutputVersionedInvokes)
|
|
|
|
|
|
|
|
|
|
program, diags, err := pcl.BindDirectory(req.SourceDirectory, loader, extraOptions...)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
if diags.HasErrors() {
|
|
|
|
|
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
|
|
|
|
|
|
|
|
|
|
return &pulumirpc.GenerateProjectResponse{
|
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
|
}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var project workspace.Project
|
|
|
|
|
if err := json.Unmarshal([]byte(req.Project), &project); err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
err = codegen.GenerateProject(req.TargetDirectory, project, program, req.LocalDependencies)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
|
|
|
|
|
|
|
|
|
|
return &pulumirpc.GenerateProjectResponse{
|
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
|
}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (host *pythonLanguageHost) GenerateProgram(
|
|
|
|
|
ctx context.Context, req *pulumirpc.GenerateProgramRequest,
|
|
|
|
|
) (*pulumirpc.GenerateProgramResponse, error) {
|
|
|
|
|
loader, err := schema.NewLoaderClient(req.LoaderTarget)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
parser := hclsyntax.NewParser()
|
|
|
|
|
// Load all .pp files in the directory
|
|
|
|
|
for path, contents := range req.Source {
|
|
|
|
|
err = parser.ParseFile(strings.NewReader(contents), path)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
diags := parser.Diagnostics
|
|
|
|
|
if diags.HasErrors() {
|
|
|
|
|
return nil, diags
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
program, diags, err := pcl.BindProgram(parser.Files,
|
|
|
|
|
pcl.Loader(loader),
|
|
|
|
|
pcl.PreferOutputVersionedInvokes)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
|
|
|
|
|
if diags.HasErrors() {
|
|
|
|
|
return &pulumirpc.GenerateProgramResponse{
|
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
|
}, nil
|
|
|
|
|
}
|
|
|
|
|
if program == nil {
|
|
|
|
|
return nil, fmt.Errorf("internal error program was nil")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
files, diags, err := codegen.GenerateProgram(program)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
rpcDiagnostics = append(rpcDiagnostics, plugin.HclDiagnosticsToRPCDiagnostics(diags)...)
|
|
|
|
|
|
|
|
|
|
return &pulumirpc.GenerateProgramResponse{
|
|
|
|
|
Source: files,
|
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
|
}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (host *pythonLanguageHost) GeneratePackage(
|
|
|
|
|
ctx context.Context, req *pulumirpc.GeneratePackageRequest,
|
|
|
|
|
) (*pulumirpc.GeneratePackageResponse, error) {
|
|
|
|
|
loader, err := schema.NewLoaderClient(req.LoaderTarget)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var spec schema.PackageSpec
|
|
|
|
|
err = json.Unmarshal([]byte(req.Schema), &spec)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pkg, diags, err := schema.BindSpec(spec, loader)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
2023-12-05 17:47:52 +00:00
|
|
|
|
rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
|
2023-08-31 16:35:21 +00:00
|
|
|
|
if diags.HasErrors() {
|
2023-12-05 17:47:52 +00:00
|
|
|
|
return &pulumirpc.GeneratePackageResponse{
|
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
|
}, nil
|
2023-08-31 16:35:21 +00:00
|
|
|
|
}
|
|
|
|
|
files, err := codegen.GeneratePackage("pulumi-language-python", pkg, req.ExtraFiles)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for filename, data := range files {
|
|
|
|
|
outPath := filepath.Join(req.Directory, filename)
|
|
|
|
|
|
|
|
|
|
err := os.MkdirAll(filepath.Dir(outPath), 0o700)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, fmt.Errorf("could not create output directory %s: %w", filepath.Dir(filename), err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
err = os.WriteFile(outPath, data, 0o600)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, fmt.Errorf("could not write output file %s: %w", filename, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-12-05 17:47:52 +00:00
|
|
|
|
return &pulumirpc.GeneratePackageResponse{
|
|
|
|
|
Diagnostics: rpcDiagnostics,
|
|
|
|
|
}, nil
|
2023-08-31 16:35:21 +00:00
|
|
|
|
}
|