pulumi/.github/workflows/ci-run-test.yml

402 lines
14 KiB
YAML
Raw Normal View History

name: Test
2022-09-02 05:40:13 +00:00
permissions:
contents: read
on:
workflow_call:
inputs:
ref:
required: true
description: "GitHub ref to use"
type: string
version:
required: true
description: "Version to produce"
type: string
platform:
description: "OS to run tests on, e.g.: ubuntu-latest"
required: true
type: string
test-name:
description: "Name of the test to run"
required: true
2022-09-02 05:40:13 +00:00
type: string
test-command:
2022-09-02 05:40:13 +00:00
description: Test command to run
required: true
type: string
is-integration-test:
description: Whether to download and install build artifacts
required: false
default: false
type: boolean
enable-coverage:
description: "Collects coverage stats; requires cov-enabled builds"
default: false
2022-09-02 05:40:13 +00:00
required: false
type: boolean
version-set:
2022-09-02 05:40:13 +00:00
required: false
description: "Set of language versions to use for builds, lints, releases, etc."
type: string
# Example provided for illustration, this value is derived by scripts/get-job-matrix.py build
default: |
{
"dotnet": "6.0.x",
"go": "1.18.x",
2023-04-11 16:20:02 +00:00
"nodejs": "16.x",
"python": "3.9.x"
}
continue-on-error:
description: "Whether to continue running the job if the step fails"
required: false
default: false
type: boolean
2022-09-02 05:40:13 +00:00
defaults:
run:
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PULUMI_VERSION: ${{ inputs.version }}
PULUMI_TEST_OWNER: "moolumi"
PULUMI_TEST_ORG: "moolumi"
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_PROD_ACCESS_TOKEN }}
# Release builds use the service, PR checks and snapshots will use the local backend if possible.
PULUMI_TEST_USE_SERVICE: ${{ !contains(inputs.version, '-') }}
Support disable plugin acquisition in deployments, and use for tests (#14104) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Missed this in the initial PR for https://github.com/pulumi/pulumi/pull/14083. This stops the deployment engine trying to install missing plugins on startup. We're also using this for CI tests for now because deploytest tries to auto install providers that don't really exist (like pkgA). Long term we'll abstract out that code so deploytest can fake the plugin cache. Fixes #14106 ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2023-10-06 10:23:14 +00:00
# We're hitting a lot of github limits because of deploytest trying to auto install plugins, skip that for now.
PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION: "true"
2022-09-02 05:40:13 +00:00
PYTHON: python
2022-09-14 06:20:01 +00:00
GO_TEST_PARALLELISM: 8
GO_TEST_PKG_PARALLELISM: 2
GO_TEST_SHUFFLE: off
PULUMI_TEST_RETRIES: 2
2022-09-02 05:40:13 +00:00
DOTNET_CLI_TELEMETRY_OPTOUT: "true"
DOTNET_ROLL_FORWARD: "Major"
SEGMENT_DOWNLOAD_TIMEOUT_MIN: 10
jobs:
test:
name: ${{ inputs.test-name }}
2022-09-02 05:40:13 +00:00
env:
PULUMI_HOME: ${{ github.workspace }}/home
TEST_ALL_DEPS: ""
runs-on: ${{ inputs.platform }}
timeout-minutes: 60
continue-on-error: ${{ inputs.continue-on-error }}
2022-09-02 05:40:13 +00:00
steps:
# Create a coverage file to register a testing job was started.
# This artifact will be overwritten with "OK" at the end of the job.
- name: Write test started file
shell: bash
run: |
HASH=$(echo '${{ inputs.test-name }}' | git hash-object --literally --stdin | awk '{ print $1 }')
echo "TEST_HASH_ID=${HASH}" >> "$GITHUB_ENV"
echo "TEST_STATUS_FILE=status/test-${HASH}.txt" >> "$GITHUB_ENV"
# Consume env.TEST_STATUS_FILE to ensure it's able to be successfully used later.
- shell: bash
run: |
mkdir status
echo "incomplete" > "${{ env.TEST_STATUS_FILE }}"
- name: Register test started.
uses: actions/upload-artifact@v3
with:
path: ${{ env.TEST_STATUS_FILE }}
2022-09-02 05:40:13 +00:00
- name: "Windows cache workaround"
# https://github.com/actions/cache/issues/752#issuecomment-1222415717
# but only modify the path by adding tar.exe
if: ${{ runner.os == 'Windows' }}
2022-09-02 05:40:13 +00:00
env:
TOOL_BIN: ${{ runner.temp }}/tar-bin
run: |
set -x
mkdir -p "${TOOL_BIN}"
cp "C:/Program Files/Git/usr/bin/tar.exe" "${TOOL_BIN}"
PATH="${TOOL_BIN}:${PATH}"
echo "$TOOL_BIN" | tee -a "$GITHUB_PATH"
command -v tar
tar --version
2022-11-03 19:49:59 +00:00
- name: Reduce Windows test parallelism
if: ${{ runner.os == 'Windows' }}
run: |
{
echo "GO_TEST_PARALLELISM=4"
echo "GO_TEST_PKG_PARALLELISM=1"
echo "GO_TEST_RACE=false"
} >> "$GITHUB_ENV"
2022-11-03 19:49:59 +00:00
# For debugging:
ps aux
2022-09-14 06:20:01 +00:00
- name: "macOS use coreutils"
if: ${{ runner.os == 'macOS' }}
run: |
set -x
brew install coreutils
echo "/usr/local/opt/coreutils/libexec/gnubin" | tee -a "$GITHUB_PATH"
command -v bash
bash --version
2022-09-02 05:40:13 +00:00
- uses: actions/checkout@v3
with:
ref: ${{ inputs.ref }}
- name: Setup versioning env vars
env:
version: ${{ inputs.version }}
run: |
./scripts/versions.sh | tee -a "$GITHUB_ENV"
- name: Enable code coverage
ci: Track code coverage **Overview** This re-enables tracking of code coverage. For Go, there are two kinds of coverage at play: unit test and integration test coverage. Unit tests follow the usual pattern of running `go test -cover -coverprofile=whatever.cov`. For integration tests, we use the new integration test profiling support [added in Go 1.20](https://go.dev/testing/coverage/). In short, the way it works is: # Build a coverage instrumented binary: go build -cover # Set GOCOVERDIR to a directory and run the integration tests # that will invoke this coverage-instrumented binary. GOCOVERDIR=$(pwd)/coverage go test ./tests # $GOCOVERDIR will now be filled with coverage data # from every invocation of the coverage-instrumented binary. # Combine it into a single coverage file: go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov # The resulting file can be uploaded to codecov as-is. The above replaces the prior, partially working hacks we had in place to get coverage-instrumented binaries with `go test -c` and hijacking the TestMain. **Notable changes** - TestMain hijacking is deleted from the Pulumi CLI. We no longer need this to build coverage-instrumented binaries. - ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH because the Pulumi binary no longer accepts a test.coverprofile flag. This information is now in the GOCOVERDIR environment variable. - We add an `enable-coverage` parameter to the `ci-build-binaries` workflow to mirror some of the other workflows. It will produce coverage-instrumented binaries if this is true. These binaries are then used by `ci-run-test` which will set `GOCOVERDIR` and merge the coverage results from it. - Coverage configuration no longer counts tests, testdata, and Protobuf-generated code against coverage. - go-wrapper.sh: Because we're no longer relying on the `go test -c` hack, this no longer excludes Windows and language providers from coverage tracking. - go-test.py and go-wrapper.sh will include pulumi-language-go and pulumi-language-nodejs in covered packages. *Other changes* - go-test.py: Fixed a bug where `args` parameters added for coverage were ignored. Note that this change DOES NOT track coverage for calls made to Pulumi packages by plugins downloaded from external sources, e.g. provider plugins. Arguably, that's out of scope of coverage trackcing for the Pulumi repository. Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
if: ${{ inputs.enable-coverage }}
2022-09-02 05:40:13 +00:00
run: |
echo "PULUMI_TEST_COVERAGE_PATH=$(pwd)/coverage" >> "$GITHUB_ENV"
ci: Track code coverage **Overview** This re-enables tracking of code coverage. For Go, there are two kinds of coverage at play: unit test and integration test coverage. Unit tests follow the usual pattern of running `go test -cover -coverprofile=whatever.cov`. For integration tests, we use the new integration test profiling support [added in Go 1.20](https://go.dev/testing/coverage/). In short, the way it works is: # Build a coverage instrumented binary: go build -cover # Set GOCOVERDIR to a directory and run the integration tests # that will invoke this coverage-instrumented binary. GOCOVERDIR=$(pwd)/coverage go test ./tests # $GOCOVERDIR will now be filled with coverage data # from every invocation of the coverage-instrumented binary. # Combine it into a single coverage file: go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov # The resulting file can be uploaded to codecov as-is. The above replaces the prior, partially working hacks we had in place to get coverage-instrumented binaries with `go test -c` and hijacking the TestMain. **Notable changes** - TestMain hijacking is deleted from the Pulumi CLI. We no longer need this to build coverage-instrumented binaries. - ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH because the Pulumi binary no longer accepts a test.coverprofile flag. This information is now in the GOCOVERDIR environment variable. - We add an `enable-coverage` parameter to the `ci-build-binaries` workflow to mirror some of the other workflows. It will produce coverage-instrumented binaries if this is true. These binaries are then used by `ci-run-test` which will set `GOCOVERDIR` and merge the coverage results from it. - Coverage configuration no longer counts tests, testdata, and Protobuf-generated code against coverage. - go-wrapper.sh: Because we're no longer relying on the `go test -c` hack, this no longer excludes Windows and language providers from coverage tracking. - go-test.py and go-wrapper.sh will include pulumi-language-go and pulumi-language-nodejs in covered packages. *Other changes* - go-test.py: Fixed a bug where `args` parameters added for coverage were ignored. Note that this change DOES NOT track coverage for calls made to Pulumi packages by plugins downloaded from external sources, e.g. provider plugins. Arguably, that's out of scope of coverage trackcing for the Pulumi repository. Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
# Post integration test coverage to a temporary directory.
# This will be merged at a later step.
echo "GOCOVERDIR=$(mktemp -d)" >> "$GITHUB_ENV"
- name: Configure Go Cache Key
2022-09-02 05:40:13 +00:00
env:
CACHE_KEY: "${{ fromJson(inputs.version-set).go }}-${{ runner.os }}-${{ runner.arch }}"
2022-09-02 05:40:13 +00:00
run: echo "$CACHE_KEY" > .gocache.tmp
- uses: actions/setup-go@v5
2022-09-02 05:40:13 +00:00
id: setup-go
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2
2022-09-02 05:40:13 +00:00
with:
go-version: ${{ fromJson(inputs.version-set).go }}
2022-09-02 05:40:13 +00:00
cache: true
cache-dependency-path: |
pkg/go.sum
.gocache.tmp
2022-09-02 05:40:13 +00:00
- name: Prime Go cache
if: ${{ ! steps.setup-go.outputs.cache-hit }}
2022-09-02 05:40:13 +00:00
# Compile every test to ensure we populate a good cache entry.
run: |
( cd sdk && go test -run "_________" ./... )
( cd pkg && go test -run "_________" ./... )
- name: Set up Python ${{ fromJson(inputs.version-set).python }}
2022-09-02 05:40:13 +00:00
uses: actions/setup-python@v3
with:
python-version: ${{ fromJson(inputs.version-set).python }}
2022-09-02 05:40:13 +00:00
cache: pip
cache-dependency-path: sdk/python/requirements.txt
- name: Set up DotNet ${{ fromJson(inputs.version-set).dotnet }}
uses: actions/setup-dotnet@v4
with:
dotnet-version: ${{ fromJson(inputs.version-set).dotnet }}
dotnet-quality: ga
- name: Set up Node ${{ fromJson(inputs.version-set).nodejs }}
2022-09-02 05:40:13 +00:00
uses: actions/setup-node@v3
with:
node-version: ${{ fromJson(inputs.version-set).nodejs }}
2022-09-02 05:40:13 +00:00
cache: yarn
cache-dependency-path: sdk/nodejs/yarn.lock
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'temurin'
- name: Setup Gradle
2023-05-15 17:58:54 +00:00
uses: gradle/gradle-build-action@v2.4.2
with:
gradle-version: "7.6"
- name: Uninstall pre-installed Pulumi (windows)
if: inputs.platform == 'windows-latest'
run: |
if command -v pulumi.exe; then
echo "Deleting pulumi"
rm -rf "$(command -v pulumi.exe)/../pulumi*"
fi
Use pnpm as package manager if we find a pnpm-lock.yaml file (#15456) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description Initial support pnpm. Note that this does not support pnpm workspaces yet. This also does not handle passing the package manager through from `pulumi new`. Once a user manually runs pnpm, creating a pnpm-lock.yaml, we'll detect that and pnpm. Fixes https://github.com/pulumi/pulumi/issues/15455 ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-02-21 13:41:21 +00:00
- name: Install yarn and pnpm
2022-09-02 05:40:13 +00:00
run: |
Use pnpm as package manager if we find a pnpm-lock.yaml file (#15456) <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description Initial support pnpm. Note that this does not support pnpm workspaces yet. This also does not handle passing the package manager through from `pulumi new`. Once a user manually runs pnpm, creating a pnpm-lock.yaml, we'll detect that and pnpm. Fixes https://github.com/pulumi/pulumi/issues/15455 ## Checklist - [x] I have run `make tidy` to update any new dependencies - [x] I have run `make lint` to verify my code passes the lint check - [x] I have formatted my code using `gofumpt` <!--- Please provide details if the checkbox below is to be left unchecked. --> - [x] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Cloud, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Cloud API version <!-- @Pulumi employees: If yes, you must submit corresponding changes in the service repo. -->
2024-02-21 13:41:21 +00:00
npm install -g yarn pnpm
2022-09-02 05:40:13 +00:00
- name: Install Python deps
run: |
python -m pip install --upgrade pip requests wheel urllib3 chardet
- name: Setup git
run: |
git config --global user.email "you@example.com"
git config --global user.name "Your Name"
- name: Set Go Dep path
run: |
echo "PULUMI_GO_DEP_ROOT=$(dirname "$(pwd)")" | tee -a "${GITHUB_ENV}"
- name: Install pulumictl
uses: jaxxstorm/action-install-gh-release@v1.7.1
env:
GITHUB_TOKEN: ${{ secrets.PULUMI_BOT_TOKEN }}
2022-09-02 05:40:13 +00:00
with:
repo: pulumi/pulumictl
2023-06-29 10:09:56 +00:00
tag: v0.0.42
2022-09-02 05:40:13 +00:00
cache: enable
- name: Install gotestsum
uses: jaxxstorm/action-install-gh-release@v1.7.1
env:
GITHUB_TOKEN: ${{ secrets.PULUMI_BOT_TOKEN }}
2022-09-02 05:40:13 +00:00
with:
repo: gotestyourself/gotestsum
tag: v1.8.1
cache: enable
- name: Install goteststats
uses: jaxxstorm/action-install-gh-release@v1.7.1
env:
GITHUB_TOKEN: ${{ secrets.PULUMI_BOT_TOKEN }}
2022-09-02 05:40:13 +00:00
with:
repo: pulumi/goteststats
2022-09-02 05:40:13 +00:00
tag: v0.0.7
cache: enable
- name: Generate artifact name
id: goenv
shell: bash
2022-09-02 05:40:13 +00:00
run: |
echo "CLI-TARGET=$(go env GOOS)-$(go env GOARCH)" >> "${GITHUB_OUTPUT}"
# Ensure tests do not rely on pre-installed packages in CI. Unit tests must run absent a local
# Pulumi install, and integration tests must only test the version built by CI.
- name: Remove Pre-installed Pulumi
env:
RUNNER_OS: ${{ runner.os }}
run: |
EXT=""
if [ "$RUNNER_OS" == "Windows" ]; then
EXT=".exe"
fi
if command -v "pulumi${EXT}"; then
PULUMI_INSTALL_DIR=$(dirname "$(command -v "pulumi${EXT}")")
echo "Deleting Pulumi"
rm -v "$PULUMI_INSTALL_DIR"/pulumi*
fi
# Integration test only steps:
2022-09-02 05:40:13 +00:00
- name: Download pulumi-${{ steps.goenv.outputs.cli-target }}
if: ${{ inputs.is-integration-test }}
2022-09-02 05:40:13 +00:00
uses: actions/download-artifact@v2
with:
name: artifacts-cli-${{ steps.goenv.outputs.cli-target }}
path: artifacts/cli
- name: Install Pulumi Go Binaries
if: ${{ inputs.is-integration-test }}
2022-09-02 05:40:13 +00:00
run: |
echo "Checking contents of $PWD/artifacts/cli"
find "$PWD/artifacts/cli"
TMPDIR="$(mktemp -d)"
mkdir -p bin
# Extract files to temporary directory:
find "$PWD/artifacts/cli" -name '*.zip' -print0 -exec unzip {} -d "$TMPDIR" \;
find "$PWD/artifacts/cli" -name '*.tar.gz' -print0 -exec tar -xzvf {} -C "$TMPDIR" \;
# Windows .zip files have an extra "bin" path part, support both:
if [ -d "$TMPDIR/pulumi/bin" ]; then
mv "${TMPDIR}/pulumi/bin/"* "$PWD/bin/"
else
mv "${TMPDIR}/pulumi/"* "$PWD/bin/"
fi
echo "Checking contents of $PWD/bin"
find "$PWD/bin"
LOCAL_PATH=$(./scripts/normpath "${{ github.workspace }}/bin")
echo "Adding LOCAL_PATH=$LOCAL_PATH to PATH"
echo "$LOCAL_PATH" >> "$GITHUB_PATH"
# /end integration test steps
- name: Verify Pulumi Version
run: |
command -v pulumi || echo "no pulumi"
pulumi version || echo "no pulumi"
- name: Ensure dependencies for the Node SDK
run: |
cd sdk/nodejs
make ensure
- name: Build the Node SDK
run: |
cd sdk/nodejs
make build_package
cd bin
yarn link
- name: Ensure dependencies for the Python SDK
run: |
cd sdk/python
make ensure
- name: Install Python SDK
run: |
cd sdk/python
make build_package
- name: Set PULUMI_ACCEPT if version-set is not current
if: ${{ fromJson(inputs.version-set).name != 'current' }}
run: echo "PULUMI_ACCEPT=TRUE" >> "${GITHUB_ENV}"
- name: Create GCP service account key file
run: |
echo '${{ secrets.GCP_SERVICE_ACCOUNT }}' > '${{ runner.temp }}/application_default_credentials.json'
- name: run tests "${{ inputs.test-command }}"
id: test
run: ${{ inputs.test-command }}
2022-09-02 05:40:13 +00:00
env:
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_STORAGE_SAS_TOKEN: ${{ secrets.AZURE_STORAGE_SAS_TOKEN }}
2022-09-02 05:40:13 +00:00
PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules
PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi
GOOGLE_APPLICATION_CREDENTIALS: ${{ runner.temp }}/application_default_credentials.json
2022-09-02 05:40:13 +00:00
- name: Convert Node coverage data
if: ${{ inputs.platform != 'windows-latest' }}
2022-09-02 05:40:13 +00:00
run: |
cd sdk/nodejs
if [ -e .nyc_output ]; then yarn run nyc report -r cobertura --report-dir "$PULUMI_TEST_COVERAGE_PATH"; fi
ci: Track code coverage **Overview** This re-enables tracking of code coverage. For Go, there are two kinds of coverage at play: unit test and integration test coverage. Unit tests follow the usual pattern of running `go test -cover -coverprofile=whatever.cov`. For integration tests, we use the new integration test profiling support [added in Go 1.20](https://go.dev/testing/coverage/). In short, the way it works is: # Build a coverage instrumented binary: go build -cover # Set GOCOVERDIR to a directory and run the integration tests # that will invoke this coverage-instrumented binary. GOCOVERDIR=$(pwd)/coverage go test ./tests # $GOCOVERDIR will now be filled with coverage data # from every invocation of the coverage-instrumented binary. # Combine it into a single coverage file: go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov # The resulting file can be uploaded to codecov as-is. The above replaces the prior, partially working hacks we had in place to get coverage-instrumented binaries with `go test -c` and hijacking the TestMain. **Notable changes** - TestMain hijacking is deleted from the Pulumi CLI. We no longer need this to build coverage-instrumented binaries. - ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH because the Pulumi binary no longer accepts a test.coverprofile flag. This information is now in the GOCOVERDIR environment variable. - We add an `enable-coverage` parameter to the `ci-build-binaries` workflow to mirror some of the other workflows. It will produce coverage-instrumented binaries if this is true. These binaries are then used by `ci-run-test` which will set `GOCOVERDIR` and merge the coverage results from it. - Coverage configuration no longer counts tests, testdata, and Protobuf-generated code against coverage. - go-wrapper.sh: Because we're no longer relying on the `go test -c` hack, this no longer excludes Windows and language providers from coverage tracking. - go-test.py and go-wrapper.sh will include pulumi-language-go and pulumi-language-nodejs in covered packages. *Other changes* - go-test.py: Fixed a bug where `args` parameters added for coverage were ignored. Note that this change DOES NOT track coverage for calls made to Pulumi packages by plugins downloaded from external sources, e.g. provider plugins. Arguably, that's out of scope of coverage trackcing for the Pulumi repository. Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
- name: Merge integration test code coverage
if: ${{ inputs.enable-coverage }}
2022-09-02 05:40:13 +00:00
run: |
ci: Track code coverage **Overview** This re-enables tracking of code coverage. For Go, there are two kinds of coverage at play: unit test and integration test coverage. Unit tests follow the usual pattern of running `go test -cover -coverprofile=whatever.cov`. For integration tests, we use the new integration test profiling support [added in Go 1.20](https://go.dev/testing/coverage/). In short, the way it works is: # Build a coverage instrumented binary: go build -cover # Set GOCOVERDIR to a directory and run the integration tests # that will invoke this coverage-instrumented binary. GOCOVERDIR=$(pwd)/coverage go test ./tests # $GOCOVERDIR will now be filled with coverage data # from every invocation of the coverage-instrumented binary. # Combine it into a single coverage file: go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov # The resulting file can be uploaded to codecov as-is. The above replaces the prior, partially working hacks we had in place to get coverage-instrumented binaries with `go test -c` and hijacking the TestMain. **Notable changes** - TestMain hijacking is deleted from the Pulumi CLI. We no longer need this to build coverage-instrumented binaries. - ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH because the Pulumi binary no longer accepts a test.coverprofile flag. This information is now in the GOCOVERDIR environment variable. - We add an `enable-coverage` parameter to the `ci-build-binaries` workflow to mirror some of the other workflows. It will produce coverage-instrumented binaries if this is true. These binaries are then used by `ci-run-test` which will set `GOCOVERDIR` and merge the coverage results from it. - Coverage configuration no longer counts tests, testdata, and Protobuf-generated code against coverage. - go-wrapper.sh: Because we're no longer relying on the `go test -c` hack, this no longer excludes Windows and language providers from coverage tracking. - go-test.py and go-wrapper.sh will include pulumi-language-go and pulumi-language-nodejs in covered packages. *Other changes* - go-test.py: Fixed a bug where `args` parameters added for coverage were ignored. Note that this change DOES NOT track coverage for calls made to Pulumi packages by plugins downloaded from external sources, e.g. provider plugins. Arguably, that's out of scope of coverage trackcing for the Pulumi repository. Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
# Merge coverage data from coverage-instrumented binaries
# if available.
if [ -n "$(ls -A "$GOCOVERDIR")" ]; then
# Cross-platform way to get milliseconds since Unix epoch.
TS=$(python -c 'import time; print(int(time.time() * 1000))')
go tool covdata textfmt -i="$GOCOVERDIR" -o="$PULUMI_TEST_COVERAGE_PATH/integration.$TS.cov"
fi
2022-09-02 05:40:13 +00:00
- name: Upload code coverage
uses: actions/upload-artifact@v2
ci: Track code coverage **Overview** This re-enables tracking of code coverage. For Go, there are two kinds of coverage at play: unit test and integration test coverage. Unit tests follow the usual pattern of running `go test -cover -coverprofile=whatever.cov`. For integration tests, we use the new integration test profiling support [added in Go 1.20](https://go.dev/testing/coverage/). In short, the way it works is: # Build a coverage instrumented binary: go build -cover # Set GOCOVERDIR to a directory and run the integration tests # that will invoke this coverage-instrumented binary. GOCOVERDIR=$(pwd)/coverage go test ./tests # $GOCOVERDIR will now be filled with coverage data # from every invocation of the coverage-instrumented binary. # Combine it into a single coverage file: go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov # The resulting file can be uploaded to codecov as-is. The above replaces the prior, partially working hacks we had in place to get coverage-instrumented binaries with `go test -c` and hijacking the TestMain. **Notable changes** - TestMain hijacking is deleted from the Pulumi CLI. We no longer need this to build coverage-instrumented binaries. - ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH because the Pulumi binary no longer accepts a test.coverprofile flag. This information is now in the GOCOVERDIR environment variable. - We add an `enable-coverage` parameter to the `ci-build-binaries` workflow to mirror some of the other workflows. It will produce coverage-instrumented binaries if this is true. These binaries are then used by `ci-run-test` which will set `GOCOVERDIR` and merge the coverage results from it. - Coverage configuration no longer counts tests, testdata, and Protobuf-generated code against coverage. - go-wrapper.sh: Because we're no longer relying on the `go test -c` hack, this no longer excludes Windows and language providers from coverage tracking. - go-test.py and go-wrapper.sh will include pulumi-language-go and pulumi-language-nodejs in covered packages. *Other changes* - go-test.py: Fixed a bug where `args` parameters added for coverage were ignored. Note that this change DOES NOT track coverage for calls made to Pulumi packages by plugins downloaded from external sources, e.g. provider plugins. Arguably, that's out of scope of coverage trackcing for the Pulumi repository. Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
if: ${{ inputs.enable-coverage }}
2022-09-02 05:40:13 +00:00
with:
name: coverage-${{ env.TEST_HASH_ID }}
path: |
coverage/*
!coverage/.gitkeep
- name: Register test OK.
shell: bash
run: mkdir status; echo "OK" > "${{ env.TEST_STATUS_FILE }}"
- name: Register test successful.
uses: actions/upload-artifact@v3
with:
path: ${{ env.TEST_STATUS_FILE }}
2022-09-02 05:40:13 +00:00
- name: Summarize Test Time by Package
2022-09-15 21:20:53 +00:00
continue-on-error: true
env:
RUNNER_OS: ${{ runner.os }}
2022-09-02 05:40:13 +00:00
run: |
mkdir -p test-results
touch test-results/empty.json # otherwise goteststats fails below when no files match
# Remove output lines, they make analysis slower & could leak logs:
if [ "$RUNNER_OS" == "macOS" ]; then
# It's just another case of BSD sed, GNU sed.
sed -i '' -e '/"Action":"output"/d' ./test-results/*.json
else
sed -i'' -e '/"Action":"output"/d' ./test-results/*.json
fi
2022-09-02 05:40:13 +00:00
goteststats -statistic pkg-time test-results/*.json
- name: Summarize Test Times by Individual Test
2022-09-15 21:20:53 +00:00
continue-on-error: true
2022-09-02 05:40:13 +00:00
run: |
goteststats -statistic test-time test-results/*.json | head -n 100 || \
if [[ $? -eq 141 ]]; then true; else exit $?; fi
- name: Upload artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v2
with:
name: gotestsum-test-results
path: test-results/*.json