2022-09-14 01:52:21 +00:00
|
|
|
name: Test
|
2022-09-02 05:40:13 +00:00
|
|
|
|
|
|
|
permissions:
|
|
|
|
contents: read
|
|
|
|
|
|
|
|
on:
|
|
|
|
workflow_call:
|
|
|
|
inputs:
|
|
|
|
ref:
|
|
|
|
required: true
|
|
|
|
description: "GitHub ref to use"
|
|
|
|
type: string
|
|
|
|
version:
|
|
|
|
required: true
|
|
|
|
description: "Version to produce"
|
|
|
|
type: string
|
|
|
|
platform:
|
|
|
|
description: "OS to run tests on, e.g.: ubuntu-latest"
|
|
|
|
required: true
|
|
|
|
type: string
|
2022-09-14 01:52:21 +00:00
|
|
|
|
|
|
|
test-name:
|
|
|
|
description: "Name of the test to run"
|
|
|
|
required: true
|
2022-09-02 05:40:13 +00:00
|
|
|
type: string
|
2022-09-14 01:52:21 +00:00
|
|
|
test-command:
|
2022-09-02 05:40:13 +00:00
|
|
|
description: Test command to run
|
|
|
|
required: true
|
|
|
|
type: string
|
|
|
|
is-integration-test:
|
|
|
|
description: Whether to download and install build artifacts
|
|
|
|
required: false
|
|
|
|
default: false
|
|
|
|
type: boolean
|
2022-09-14 01:52:21 +00:00
|
|
|
enable-coverage:
|
|
|
|
description: "Collects coverage stats; requires cov-enabled builds"
|
|
|
|
default: false
|
2022-09-02 05:40:13 +00:00
|
|
|
required: false
|
2022-09-14 01:52:21 +00:00
|
|
|
type: boolean
|
|
|
|
|
|
|
|
version-set:
|
2022-09-02 05:40:13 +00:00
|
|
|
required: false
|
2022-09-14 01:52:21 +00:00
|
|
|
description: "Set of language versions to use for builds, lints, releases, etc."
|
|
|
|
type: string
|
|
|
|
# Example provided for illustration, this value is derived by scripts/get-job-matrix.py build
|
|
|
|
default: |
|
|
|
|
{
|
|
|
|
"dotnet": "6.0.x",
|
|
|
|
"go": "1.18.x",
|
2023-04-11 16:20:02 +00:00
|
|
|
"nodejs": "16.x",
|
2022-09-14 01:52:21 +00:00
|
|
|
"python": "3.9.x"
|
|
|
|
}
|
2022-11-25 21:48:05 +00:00
|
|
|
continue-on-error:
|
|
|
|
description: "Whether to continue running the job if the step fails"
|
|
|
|
required: false
|
|
|
|
default: false
|
|
|
|
type: boolean
|
2022-09-14 01:52:21 +00:00
|
|
|
|
2022-09-02 05:40:13 +00:00
|
|
|
defaults:
|
|
|
|
run:
|
|
|
|
shell: bash
|
|
|
|
|
|
|
|
env:
|
|
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
PULUMI_VERSION: ${{ inputs.version }}
|
|
|
|
PULUMI_TEST_OWNER: "moolumi"
|
|
|
|
PULUMI_TEST_ORG: "moolumi"
|
|
|
|
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_PROD_ACCESS_TOKEN }}
|
2022-09-14 02:34:01 +00:00
|
|
|
# Release builds use the service, PR checks and snapshots will use the local backend if possible.
|
|
|
|
PULUMI_TEST_USE_SERVICE: ${{ !contains(inputs.version, '-') }}
|
2023-10-06 10:23:14 +00:00
|
|
|
# We're hitting a lot of github limits because of deploytest trying to auto install plugins, skip that for now.
|
2023-10-14 08:32:43 +00:00
|
|
|
PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION: "true"
|
2022-09-02 05:40:13 +00:00
|
|
|
PYTHON: python
|
2022-09-14 06:20:01 +00:00
|
|
|
GO_TEST_PARALLELISM: 8
|
|
|
|
GO_TEST_PKG_PARALLELISM: 2
|
|
|
|
GO_TEST_SHUFFLE: off
|
|
|
|
PULUMI_TEST_RETRIES: 2
|
2022-09-02 05:40:13 +00:00
|
|
|
DOTNET_CLI_TELEMETRY_OPTOUT: "true"
|
|
|
|
DOTNET_ROLL_FORWARD: "Major"
|
|
|
|
SEGMENT_DOWNLOAD_TIMEOUT_MIN: 10
|
|
|
|
|
|
|
|
jobs:
|
|
|
|
test:
|
2022-09-14 01:52:21 +00:00
|
|
|
name: ${{ inputs.test-name }}
|
2022-09-02 05:40:13 +00:00
|
|
|
env:
|
|
|
|
PULUMI_HOME: ${{ github.workspace }}/home
|
|
|
|
TEST_ALL_DEPS: ""
|
|
|
|
|
|
|
|
runs-on: ${{ inputs.platform }}
|
|
|
|
|
2022-11-25 21:48:05 +00:00
|
|
|
timeout-minutes: 60
|
|
|
|
continue-on-error: ${{ inputs.continue-on-error }}
|
2022-09-02 05:40:13 +00:00
|
|
|
steps:
|
2023-12-04 02:03:42 +00:00
|
|
|
# Create a coverage file to register a testing job was started.
|
|
|
|
# This artifact will be overwritten with "OK" at the end of the job.
|
|
|
|
- name: Write test started file
|
|
|
|
shell: bash
|
|
|
|
run: |
|
|
|
|
HASH=$(echo '${{ inputs.test-name }}' | git hash-object --literally --stdin | awk '{ print $1 }')
|
|
|
|
echo "TEST_HASH_ID=${HASH}" >> "$GITHUB_ENV"
|
|
|
|
echo "TEST_STATUS_FILE=status/test-${HASH}.txt" >> "$GITHUB_ENV"
|
|
|
|
# Consume env.TEST_STATUS_FILE to ensure it's able to be successfully used later.
|
|
|
|
- shell: bash
|
|
|
|
run: |
|
|
|
|
mkdir status
|
|
|
|
echo "incomplete" > "${{ env.TEST_STATUS_FILE }}"
|
|
|
|
- name: Register test started.
|
|
|
|
uses: actions/upload-artifact@v3
|
|
|
|
with:
|
|
|
|
path: ${{ env.TEST_STATUS_FILE }}
|
2022-09-02 05:40:13 +00:00
|
|
|
- name: "Windows cache workaround"
|
|
|
|
# https://github.com/actions/cache/issues/752#issuecomment-1222415717
|
|
|
|
# but only modify the path by adding tar.exe
|
2022-09-14 01:52:21 +00:00
|
|
|
if: ${{ runner.os == 'Windows' }}
|
2022-09-02 05:40:13 +00:00
|
|
|
env:
|
|
|
|
TOOL_BIN: ${{ runner.temp }}/tar-bin
|
|
|
|
run: |
|
|
|
|
set -x
|
|
|
|
mkdir -p "${TOOL_BIN}"
|
|
|
|
cp "C:/Program Files/Git/usr/bin/tar.exe" "${TOOL_BIN}"
|
|
|
|
PATH="${TOOL_BIN}:${PATH}"
|
|
|
|
echo "$TOOL_BIN" | tee -a "$GITHUB_PATH"
|
|
|
|
command -v tar
|
|
|
|
tar --version
|
2022-11-03 19:49:59 +00:00
|
|
|
- name: Reduce Windows test parallelism
|
|
|
|
if: ${{ runner.os == 'Windows' }}
|
|
|
|
run: |
|
2023-07-26 22:02:59 +00:00
|
|
|
{
|
|
|
|
echo "GO_TEST_PARALLELISM=4"
|
|
|
|
echo "GO_TEST_PKG_PARALLELISM=1"
|
|
|
|
echo "GO_TEST_RACE=false"
|
|
|
|
} >> "$GITHUB_ENV"
|
2022-11-03 19:49:59 +00:00
|
|
|
# For debugging:
|
|
|
|
ps aux
|
2022-09-14 06:20:01 +00:00
|
|
|
- name: "macOS use coreutils"
|
|
|
|
if: ${{ runner.os == 'macOS' }}
|
|
|
|
run: |
|
|
|
|
set -x
|
|
|
|
brew install coreutils
|
|
|
|
echo "/usr/local/opt/coreutils/libexec/gnubin" | tee -a "$GITHUB_PATH"
|
|
|
|
command -v bash
|
|
|
|
bash --version
|
2022-09-02 05:40:13 +00:00
|
|
|
- uses: actions/checkout@v3
|
|
|
|
with:
|
|
|
|
ref: ${{ inputs.ref }}
|
|
|
|
- name: Setup versioning env vars
|
|
|
|
env:
|
|
|
|
version: ${{ inputs.version }}
|
|
|
|
run: |
|
|
|
|
./scripts/versions.sh | tee -a "$GITHUB_ENV"
|
|
|
|
- name: Enable code coverage
|
ci: Track code coverage
**Overview**
This re-enables tracking of code coverage.
For Go, there are two kinds of coverage at play:
unit test and integration test coverage.
Unit tests follow the usual pattern of running
`go test -cover -coverprofile=whatever.cov`.
For integration tests, we use the new integration test profiling support
[added in Go 1.20](https://go.dev/testing/coverage/).
In short, the way it works is:
# Build a coverage instrumented binary:
go build -cover
# Set GOCOVERDIR to a directory and run the integration tests
# that will invoke this coverage-instrumented binary.
GOCOVERDIR=$(pwd)/coverage
go test ./tests
# $GOCOVERDIR will now be filled with coverage data
# from every invocation of the coverage-instrumented binary.
# Combine it into a single coverage file:
go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov
# The resulting file can be uploaded to codecov as-is.
The above replaces the prior, partially working hacks we had in place
to get coverage-instrumented binaries with `go test -c`
and hijacking the TestMain.
**Notable changes**
- TestMain hijacking is deleted from the Pulumi CLI.
We no longer need this to build coverage-instrumented binaries.
- ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH
because the Pulumi binary no longer accepts a test.coverprofile flag.
This information is now in the GOCOVERDIR environment variable.
- We add an `enable-coverage` parameter to the `ci-build-binaries`
workflow to mirror some of the other workflows.
It will produce coverage-instrumented binaries if this is true.
These binaries are then used by `ci-run-test` which will set
`GOCOVERDIR` and merge the coverage results from it.
- Coverage configuration no longer counts tests, testdata,
and Protobuf-generated code against coverage.
- go-wrapper.sh:
Because we're no longer relying on the `go test -c` hack,
this no longer excludes Windows and language providers
from coverage tracking.
- go-test.py and go-wrapper.sh will include pulumi-language-go and
pulumi-language-nodejs in covered packages.
*Other changes*
- go-test.py:
Fixed a bug where `args` parameters added for coverage were ignored.
Note that this change DOES NOT track coverage for calls made to Pulumi
packages by plugins downloaded from external sources,
e.g. provider plugins. Arguably, that's out of scope of coverage
trackcing for the Pulumi repository.
Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
|
|
|
if: ${{ inputs.enable-coverage }}
|
2022-09-02 05:40:13 +00:00
|
|
|
run: |
|
|
|
|
echo "PULUMI_TEST_COVERAGE_PATH=$(pwd)/coverage" >> "$GITHUB_ENV"
|
ci: Track code coverage
**Overview**
This re-enables tracking of code coverage.
For Go, there are two kinds of coverage at play:
unit test and integration test coverage.
Unit tests follow the usual pattern of running
`go test -cover -coverprofile=whatever.cov`.
For integration tests, we use the new integration test profiling support
[added in Go 1.20](https://go.dev/testing/coverage/).
In short, the way it works is:
# Build a coverage instrumented binary:
go build -cover
# Set GOCOVERDIR to a directory and run the integration tests
# that will invoke this coverage-instrumented binary.
GOCOVERDIR=$(pwd)/coverage
go test ./tests
# $GOCOVERDIR will now be filled with coverage data
# from every invocation of the coverage-instrumented binary.
# Combine it into a single coverage file:
go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov
# The resulting file can be uploaded to codecov as-is.
The above replaces the prior, partially working hacks we had in place
to get coverage-instrumented binaries with `go test -c`
and hijacking the TestMain.
**Notable changes**
- TestMain hijacking is deleted from the Pulumi CLI.
We no longer need this to build coverage-instrumented binaries.
- ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH
because the Pulumi binary no longer accepts a test.coverprofile flag.
This information is now in the GOCOVERDIR environment variable.
- We add an `enable-coverage` parameter to the `ci-build-binaries`
workflow to mirror some of the other workflows.
It will produce coverage-instrumented binaries if this is true.
These binaries are then used by `ci-run-test` which will set
`GOCOVERDIR` and merge the coverage results from it.
- Coverage configuration no longer counts tests, testdata,
and Protobuf-generated code against coverage.
- go-wrapper.sh:
Because we're no longer relying on the `go test -c` hack,
this no longer excludes Windows and language providers
from coverage tracking.
- go-test.py and go-wrapper.sh will include pulumi-language-go and
pulumi-language-nodejs in covered packages.
*Other changes*
- go-test.py:
Fixed a bug where `args` parameters added for coverage were ignored.
Note that this change DOES NOT track coverage for calls made to Pulumi
packages by plugins downloaded from external sources,
e.g. provider plugins. Arguably, that's out of scope of coverage
trackcing for the Pulumi repository.
Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
|
|
|
# Post integration test coverage to a temporary directory.
|
|
|
|
# This will be merged at a later step.
|
|
|
|
echo "GOCOVERDIR=$(mktemp -d)" >> "$GITHUB_ENV"
|
2022-09-14 01:52:21 +00:00
|
|
|
- name: Configure Go Cache Key
|
2022-09-02 05:40:13 +00:00
|
|
|
env:
|
2022-09-14 01:52:21 +00:00
|
|
|
CACHE_KEY: "${{ fromJson(inputs.version-set).go }}-${{ runner.os }}-${{ runner.arch }}"
|
2022-09-02 05:40:13 +00:00
|
|
|
run: echo "$CACHE_KEY" > .gocache.tmp
|
2023-12-08 20:58:56 +00:00
|
|
|
- uses: actions/setup-go@v5
|
2022-09-02 05:40:13 +00:00
|
|
|
id: setup-go
|
2023-12-08 20:58:56 +00:00
|
|
|
env:
|
|
|
|
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2
|
2022-09-02 05:40:13 +00:00
|
|
|
with:
|
2022-09-14 01:52:21 +00:00
|
|
|
go-version: ${{ fromJson(inputs.version-set).go }}
|
2022-09-02 05:40:13 +00:00
|
|
|
cache: true
|
2023-07-06 17:37:08 +00:00
|
|
|
cache-dependency-path: |
|
|
|
|
pkg/go.sum
|
|
|
|
.gocache.tmp
|
2022-09-02 05:40:13 +00:00
|
|
|
- name: Prime Go cache
|
2024-03-18 14:04:12 +00:00
|
|
|
if: ${{ ! steps.setup-go.outputs.cache-hit }}
|
2022-09-02 05:40:13 +00:00
|
|
|
# Compile every test to ensure we populate a good cache entry.
|
|
|
|
run: |
|
|
|
|
( cd sdk && go test -run "_________" ./... )
|
|
|
|
( cd pkg && go test -run "_________" ./... )
|
2022-09-14 01:52:21 +00:00
|
|
|
- name: Set up Python ${{ fromJson(inputs.version-set).python }}
|
2022-09-02 05:40:13 +00:00
|
|
|
uses: actions/setup-python@v3
|
|
|
|
with:
|
2022-09-14 01:52:21 +00:00
|
|
|
python-version: ${{ fromJson(inputs.version-set).python }}
|
2022-09-02 05:40:13 +00:00
|
|
|
cache: pip
|
|
|
|
cache-dependency-path: sdk/python/requirements.txt
|
2022-11-11 19:27:25 +00:00
|
|
|
- name: Set up DotNet ${{ fromJson(inputs.version-set).dotnet }}
|
2024-02-20 12:20:04 +00:00
|
|
|
uses: actions/setup-dotnet@v4
|
2022-11-11 19:27:25 +00:00
|
|
|
with:
|
|
|
|
dotnet-version: ${{ fromJson(inputs.version-set).dotnet }}
|
|
|
|
dotnet-quality: ga
|
2022-09-14 01:52:21 +00:00
|
|
|
- name: Set up Node ${{ fromJson(inputs.version-set).nodejs }}
|
2022-09-02 05:40:13 +00:00
|
|
|
uses: actions/setup-node@v3
|
|
|
|
with:
|
2022-09-14 01:52:21 +00:00
|
|
|
node-version: ${{ fromJson(inputs.version-set).nodejs }}
|
2022-09-02 05:40:13 +00:00
|
|
|
cache: yarn
|
2022-09-17 02:25:06 +00:00
|
|
|
cache-dependency-path: sdk/nodejs/yarn.lock
|
2023-05-04 18:59:37 +00:00
|
|
|
- name: Set up JDK 11
|
|
|
|
uses: actions/setup-java@v3
|
|
|
|
with:
|
|
|
|
java-version: '11'
|
|
|
|
distribution: 'temurin'
|
|
|
|
- name: Setup Gradle
|
2023-05-15 17:58:54 +00:00
|
|
|
uses: gradle/gradle-build-action@v2.4.2
|
2023-05-04 18:59:37 +00:00
|
|
|
with:
|
|
|
|
gradle-version: "7.6"
|
2022-09-17 02:25:06 +00:00
|
|
|
- name: Uninstall pre-installed Pulumi (windows)
|
|
|
|
if: inputs.platform == 'windows-latest'
|
|
|
|
run: |
|
|
|
|
if command -v pulumi.exe; then
|
|
|
|
echo "Deleting pulumi"
|
|
|
|
rm -rf "$(command -v pulumi.exe)/../pulumi*"
|
|
|
|
fi
|
2024-02-21 13:41:21 +00:00
|
|
|
- name: Install yarn and pnpm
|
2022-09-02 05:40:13 +00:00
|
|
|
run: |
|
2024-02-21 13:41:21 +00:00
|
|
|
npm install -g yarn pnpm
|
2022-09-02 05:40:13 +00:00
|
|
|
- name: Install Python deps
|
|
|
|
run: |
|
|
|
|
python -m pip install --upgrade pip requests wheel urllib3 chardet
|
|
|
|
- name: Setup git
|
|
|
|
run: |
|
|
|
|
git config --global user.email "you@example.com"
|
|
|
|
git config --global user.name "Your Name"
|
|
|
|
- name: Set Go Dep path
|
|
|
|
run: |
|
|
|
|
echo "PULUMI_GO_DEP_ROOT=$(dirname "$(pwd)")" | tee -a "${GITHUB_ENV}"
|
|
|
|
- name: Install pulumictl
|
|
|
|
uses: jaxxstorm/action-install-gh-release@v1.7.1
|
2023-03-02 01:04:51 +00:00
|
|
|
env:
|
|
|
|
GITHUB_TOKEN: ${{ secrets.PULUMI_BOT_TOKEN }}
|
2022-09-02 05:40:13 +00:00
|
|
|
with:
|
|
|
|
repo: pulumi/pulumictl
|
2023-06-29 10:09:56 +00:00
|
|
|
tag: v0.0.42
|
2022-09-02 05:40:13 +00:00
|
|
|
cache: enable
|
|
|
|
- name: Install gotestsum
|
|
|
|
uses: jaxxstorm/action-install-gh-release@v1.7.1
|
2023-03-02 01:04:51 +00:00
|
|
|
env:
|
|
|
|
GITHUB_TOKEN: ${{ secrets.PULUMI_BOT_TOKEN }}
|
2022-09-02 05:40:13 +00:00
|
|
|
with:
|
|
|
|
repo: gotestyourself/gotestsum
|
|
|
|
tag: v1.8.1
|
|
|
|
cache: enable
|
|
|
|
- name: Install goteststats
|
|
|
|
uses: jaxxstorm/action-install-gh-release@v1.7.1
|
2023-03-02 01:04:51 +00:00
|
|
|
env:
|
|
|
|
GITHUB_TOKEN: ${{ secrets.PULUMI_BOT_TOKEN }}
|
2022-09-02 05:40:13 +00:00
|
|
|
with:
|
2023-08-10 20:54:14 +00:00
|
|
|
repo: pulumi/goteststats
|
2022-09-02 05:40:13 +00:00
|
|
|
tag: v0.0.7
|
|
|
|
cache: enable
|
|
|
|
- name: Generate artifact name
|
|
|
|
id: goenv
|
2023-02-18 13:55:39 +00:00
|
|
|
shell: bash
|
2022-09-02 05:40:13 +00:00
|
|
|
run: |
|
2023-02-18 13:55:39 +00:00
|
|
|
echo "CLI-TARGET=$(go env GOOS)-$(go env GOARCH)" >> "${GITHUB_OUTPUT}"
|
2022-09-14 01:52:21 +00:00
|
|
|
|
|
|
|
# Ensure tests do not rely on pre-installed packages in CI. Unit tests must run absent a local
|
|
|
|
# Pulumi install, and integration tests must only test the version built by CI.
|
|
|
|
- name: Remove Pre-installed Pulumi
|
|
|
|
env:
|
|
|
|
RUNNER_OS: ${{ runner.os }}
|
|
|
|
run: |
|
|
|
|
EXT=""
|
|
|
|
if [ "$RUNNER_OS" == "Windows" ]; then
|
|
|
|
EXT=".exe"
|
|
|
|
fi
|
|
|
|
|
|
|
|
if command -v "pulumi${EXT}"; then
|
|
|
|
PULUMI_INSTALL_DIR=$(dirname "$(command -v "pulumi${EXT}")")
|
|
|
|
echo "Deleting Pulumi"
|
|
|
|
rm -v "$PULUMI_INSTALL_DIR"/pulumi*
|
|
|
|
fi
|
|
|
|
|
|
|
|
# Integration test only steps:
|
2022-09-02 05:40:13 +00:00
|
|
|
- name: Download pulumi-${{ steps.goenv.outputs.cli-target }}
|
2022-09-14 01:52:21 +00:00
|
|
|
if: ${{ inputs.is-integration-test }}
|
2022-09-02 05:40:13 +00:00
|
|
|
uses: actions/download-artifact@v2
|
|
|
|
with:
|
|
|
|
name: artifacts-cli-${{ steps.goenv.outputs.cli-target }}
|
|
|
|
path: artifacts/cli
|
|
|
|
- name: Install Pulumi Go Binaries
|
2022-09-14 01:52:21 +00:00
|
|
|
if: ${{ inputs.is-integration-test }}
|
2022-09-02 05:40:13 +00:00
|
|
|
run: |
|
|
|
|
echo "Checking contents of $PWD/artifacts/cli"
|
|
|
|
find "$PWD/artifacts/cli"
|
|
|
|
TMPDIR="$(mktemp -d)"
|
|
|
|
|
|
|
|
mkdir -p bin
|
|
|
|
|
|
|
|
# Extract files to temporary directory:
|
|
|
|
find "$PWD/artifacts/cli" -name '*.zip' -print0 -exec unzip {} -d "$TMPDIR" \;
|
|
|
|
find "$PWD/artifacts/cli" -name '*.tar.gz' -print0 -exec tar -xzvf {} -C "$TMPDIR" \;
|
|
|
|
|
|
|
|
# Windows .zip files have an extra "bin" path part, support both:
|
|
|
|
if [ -d "$TMPDIR/pulumi/bin" ]; then
|
|
|
|
mv "${TMPDIR}/pulumi/bin/"* "$PWD/bin/"
|
|
|
|
else
|
|
|
|
mv "${TMPDIR}/pulumi/"* "$PWD/bin/"
|
|
|
|
fi
|
|
|
|
|
|
|
|
echo "Checking contents of $PWD/bin"
|
|
|
|
find "$PWD/bin"
|
|
|
|
|
|
|
|
LOCAL_PATH=$(./scripts/normpath "${{ github.workspace }}/bin")
|
|
|
|
echo "Adding LOCAL_PATH=$LOCAL_PATH to PATH"
|
|
|
|
echo "$LOCAL_PATH" >> "$GITHUB_PATH"
|
|
|
|
|
|
|
|
# /end integration test steps
|
|
|
|
- name: Verify Pulumi Version
|
|
|
|
run: |
|
|
|
|
command -v pulumi || echo "no pulumi"
|
|
|
|
pulumi version || echo "no pulumi"
|
|
|
|
- name: Ensure dependencies for the Node SDK
|
|
|
|
run: |
|
|
|
|
cd sdk/nodejs
|
|
|
|
make ensure
|
|
|
|
- name: Build the Node SDK
|
|
|
|
run: |
|
|
|
|
cd sdk/nodejs
|
|
|
|
make build_package
|
|
|
|
cd bin
|
|
|
|
yarn link
|
|
|
|
- name: Ensure dependencies for the Python SDK
|
|
|
|
run: |
|
|
|
|
cd sdk/python
|
|
|
|
make ensure
|
|
|
|
- name: Install Python SDK
|
|
|
|
run: |
|
|
|
|
cd sdk/python
|
|
|
|
make build_package
|
2022-09-14 01:52:21 +00:00
|
|
|
- name: Set PULUMI_ACCEPT if version-set is not current
|
|
|
|
if: ${{ fromJson(inputs.version-set).name != 'current' }}
|
|
|
|
run: echo "PULUMI_ACCEPT=TRUE" >> "${GITHUB_ENV}"
|
2024-01-03 14:47:09 +00:00
|
|
|
- name: Create GCP service account key file
|
|
|
|
run: |
|
|
|
|
echo '${{ secrets.GCP_SERVICE_ACCOUNT }}' > '${{ runner.temp }}/application_default_credentials.json'
|
2024-01-04 18:12:29 +00:00
|
|
|
- name: run tests "${{ inputs.test-command }}"
|
2022-09-14 01:52:21 +00:00
|
|
|
id: test
|
2023-10-06 13:54:28 +00:00
|
|
|
run: ${{ inputs.test-command }}
|
2022-09-02 05:40:13 +00:00
|
|
|
env:
|
2024-01-03 14:47:09 +00:00
|
|
|
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
|
|
|
|
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
|
|
|
|
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
|
2024-01-23 09:41:20 +00:00
|
|
|
AZURE_STORAGE_SAS_TOKEN: ${{ secrets.AZURE_STORAGE_SAS_TOKEN }}
|
2022-09-02 05:40:13 +00:00
|
|
|
PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules
|
|
|
|
PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi
|
2024-01-03 14:47:09 +00:00
|
|
|
GOOGLE_APPLICATION_CREDENTIALS: ${{ runner.temp }}/application_default_credentials.json
|
2022-09-02 05:40:13 +00:00
|
|
|
- name: Convert Node coverage data
|
2022-09-14 01:52:21 +00:00
|
|
|
if: ${{ inputs.platform != 'windows-latest' }}
|
2022-09-02 05:40:13 +00:00
|
|
|
run: |
|
|
|
|
cd sdk/nodejs
|
|
|
|
if [ -e .nyc_output ]; then yarn run nyc report -r cobertura --report-dir "$PULUMI_TEST_COVERAGE_PATH"; fi
|
ci: Track code coverage
**Overview**
This re-enables tracking of code coverage.
For Go, there are two kinds of coverage at play:
unit test and integration test coverage.
Unit tests follow the usual pattern of running
`go test -cover -coverprofile=whatever.cov`.
For integration tests, we use the new integration test profiling support
[added in Go 1.20](https://go.dev/testing/coverage/).
In short, the way it works is:
# Build a coverage instrumented binary:
go build -cover
# Set GOCOVERDIR to a directory and run the integration tests
# that will invoke this coverage-instrumented binary.
GOCOVERDIR=$(pwd)/coverage
go test ./tests
# $GOCOVERDIR will now be filled with coverage data
# from every invocation of the coverage-instrumented binary.
# Combine it into a single coverage file:
go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov
# The resulting file can be uploaded to codecov as-is.
The above replaces the prior, partially working hacks we had in place
to get coverage-instrumented binaries with `go test -c`
and hijacking the TestMain.
**Notable changes**
- TestMain hijacking is deleted from the Pulumi CLI.
We no longer need this to build coverage-instrumented binaries.
- ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH
because the Pulumi binary no longer accepts a test.coverprofile flag.
This information is now in the GOCOVERDIR environment variable.
- We add an `enable-coverage` parameter to the `ci-build-binaries`
workflow to mirror some of the other workflows.
It will produce coverage-instrumented binaries if this is true.
These binaries are then used by `ci-run-test` which will set
`GOCOVERDIR` and merge the coverage results from it.
- Coverage configuration no longer counts tests, testdata,
and Protobuf-generated code against coverage.
- go-wrapper.sh:
Because we're no longer relying on the `go test -c` hack,
this no longer excludes Windows and language providers
from coverage tracking.
- go-test.py and go-wrapper.sh will include pulumi-language-go and
pulumi-language-nodejs in covered packages.
*Other changes*
- go-test.py:
Fixed a bug where `args` parameters added for coverage were ignored.
Note that this change DOES NOT track coverage for calls made to Pulumi
packages by plugins downloaded from external sources,
e.g. provider plugins. Arguably, that's out of scope of coverage
trackcing for the Pulumi repository.
Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
|
|
|
- name: Merge integration test code coverage
|
|
|
|
if: ${{ inputs.enable-coverage }}
|
2022-09-02 05:40:13 +00:00
|
|
|
run: |
|
ci: Track code coverage
**Overview**
This re-enables tracking of code coverage.
For Go, there are two kinds of coverage at play:
unit test and integration test coverage.
Unit tests follow the usual pattern of running
`go test -cover -coverprofile=whatever.cov`.
For integration tests, we use the new integration test profiling support
[added in Go 1.20](https://go.dev/testing/coverage/).
In short, the way it works is:
# Build a coverage instrumented binary:
go build -cover
# Set GOCOVERDIR to a directory and run the integration tests
# that will invoke this coverage-instrumented binary.
GOCOVERDIR=$(pwd)/coverage
go test ./tests
# $GOCOVERDIR will now be filled with coverage data
# from every invocation of the coverage-instrumented binary.
# Combine it into a single coverage file:
go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov
# The resulting file can be uploaded to codecov as-is.
The above replaces the prior, partially working hacks we had in place
to get coverage-instrumented binaries with `go test -c`
and hijacking the TestMain.
**Notable changes**
- TestMain hijacking is deleted from the Pulumi CLI.
We no longer need this to build coverage-instrumented binaries.
- ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH
because the Pulumi binary no longer accepts a test.coverprofile flag.
This information is now in the GOCOVERDIR environment variable.
- We add an `enable-coverage` parameter to the `ci-build-binaries`
workflow to mirror some of the other workflows.
It will produce coverage-instrumented binaries if this is true.
These binaries are then used by `ci-run-test` which will set
`GOCOVERDIR` and merge the coverage results from it.
- Coverage configuration no longer counts tests, testdata,
and Protobuf-generated code against coverage.
- go-wrapper.sh:
Because we're no longer relying on the `go test -c` hack,
this no longer excludes Windows and language providers
from coverage tracking.
- go-test.py and go-wrapper.sh will include pulumi-language-go and
pulumi-language-nodejs in covered packages.
*Other changes*
- go-test.py:
Fixed a bug where `args` parameters added for coverage were ignored.
Note that this change DOES NOT track coverage for calls made to Pulumi
packages by plugins downloaded from external sources,
e.g. provider plugins. Arguably, that's out of scope of coverage
trackcing for the Pulumi repository.
Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
|
|
|
# Merge coverage data from coverage-instrumented binaries
|
|
|
|
# if available.
|
|
|
|
if [ -n "$(ls -A "$GOCOVERDIR")" ]; then
|
|
|
|
# Cross-platform way to get milliseconds since Unix epoch.
|
|
|
|
TS=$(python -c 'import time; print(int(time.time() * 1000))')
|
|
|
|
go tool covdata textfmt -i="$GOCOVERDIR" -o="$PULUMI_TEST_COVERAGE_PATH/integration.$TS.cov"
|
|
|
|
fi
|
2022-09-02 05:40:13 +00:00
|
|
|
- name: Upload code coverage
|
2023-12-04 02:03:42 +00:00
|
|
|
uses: actions/upload-artifact@v2
|
ci: Track code coverage
**Overview**
This re-enables tracking of code coverage.
For Go, there are two kinds of coverage at play:
unit test and integration test coverage.
Unit tests follow the usual pattern of running
`go test -cover -coverprofile=whatever.cov`.
For integration tests, we use the new integration test profiling support
[added in Go 1.20](https://go.dev/testing/coverage/).
In short, the way it works is:
# Build a coverage instrumented binary:
go build -cover
# Set GOCOVERDIR to a directory and run the integration tests
# that will invoke this coverage-instrumented binary.
GOCOVERDIR=$(pwd)/coverage
go test ./tests
# $GOCOVERDIR will now be filled with coverage data
# from every invocation of the coverage-instrumented binary.
# Combine it into a single coverage file:
go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov
# The resulting file can be uploaded to codecov as-is.
The above replaces the prior, partially working hacks we had in place
to get coverage-instrumented binaries with `go test -c`
and hijacking the TestMain.
**Notable changes**
- TestMain hijacking is deleted from the Pulumi CLI.
We no longer need this to build coverage-instrumented binaries.
- ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH
because the Pulumi binary no longer accepts a test.coverprofile flag.
This information is now in the GOCOVERDIR environment variable.
- We add an `enable-coverage` parameter to the `ci-build-binaries`
workflow to mirror some of the other workflows.
It will produce coverage-instrumented binaries if this is true.
These binaries are then used by `ci-run-test` which will set
`GOCOVERDIR` and merge the coverage results from it.
- Coverage configuration no longer counts tests, testdata,
and Protobuf-generated code against coverage.
- go-wrapper.sh:
Because we're no longer relying on the `go test -c` hack,
this no longer excludes Windows and language providers
from coverage tracking.
- go-test.py and go-wrapper.sh will include pulumi-language-go and
pulumi-language-nodejs in covered packages.
*Other changes*
- go-test.py:
Fixed a bug where `args` parameters added for coverage were ignored.
Note that this change DOES NOT track coverage for calls made to Pulumi
packages by plugins downloaded from external sources,
e.g. provider plugins. Arguably, that's out of scope of coverage
trackcing for the Pulumi repository.
Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
|
|
|
if: ${{ inputs.enable-coverage }}
|
2022-09-02 05:40:13 +00:00
|
|
|
with:
|
2023-12-04 02:03:42 +00:00
|
|
|
name: coverage-${{ env.TEST_HASH_ID }}
|
|
|
|
path: |
|
|
|
|
coverage/*
|
|
|
|
!coverage/.gitkeep
|
|
|
|
- name: Register test OK.
|
|
|
|
shell: bash
|
|
|
|
run: mkdir status; echo "OK" > "${{ env.TEST_STATUS_FILE }}"
|
|
|
|
- name: Register test successful.
|
|
|
|
uses: actions/upload-artifact@v3
|
|
|
|
with:
|
|
|
|
path: ${{ env.TEST_STATUS_FILE }}
|
2022-09-02 05:40:13 +00:00
|
|
|
- name: Summarize Test Time by Package
|
2022-09-15 21:20:53 +00:00
|
|
|
continue-on-error: true
|
2022-09-14 01:52:21 +00:00
|
|
|
env:
|
|
|
|
RUNNER_OS: ${{ runner.os }}
|
2022-09-02 05:40:13 +00:00
|
|
|
run: |
|
|
|
|
mkdir -p test-results
|
|
|
|
touch test-results/empty.json # otherwise goteststats fails below when no files match
|
2022-09-14 01:52:21 +00:00
|
|
|
# Remove output lines, they make analysis slower & could leak logs:
|
|
|
|
if [ "$RUNNER_OS" == "macOS" ]; then
|
|
|
|
# It's just another case of BSD sed, GNU sed.
|
|
|
|
sed -i '' -e '/"Action":"output"/d' ./test-results/*.json
|
|
|
|
else
|
|
|
|
sed -i'' -e '/"Action":"output"/d' ./test-results/*.json
|
|
|
|
fi
|
|
|
|
|
2022-09-02 05:40:13 +00:00
|
|
|
goteststats -statistic pkg-time test-results/*.json
|
|
|
|
- name: Summarize Test Times by Individual Test
|
2022-09-15 21:20:53 +00:00
|
|
|
continue-on-error: true
|
2022-09-02 05:40:13 +00:00
|
|
|
run: |
|
|
|
|
goteststats -statistic test-time test-results/*.json | head -n 100 || \
|
|
|
|
if [[ $? -eq 141 ]]; then true; else exit $?; fi
|
2022-09-14 01:52:21 +00:00
|
|
|
- name: Upload artifacts
|
|
|
|
if: ${{ always() }}
|
|
|
|
uses: actions/upload-artifact@v2
|
|
|
|
with:
|
|
|
|
name: gotestsum-test-results
|
|
|
|
path: test-results/*.json
|