pulumi/.github/workflows/ci.yml

482 lines
19 KiB
YAML
Raw Normal View History

2022-09-02 05:40:13 +00:00
name: CI
permissions:
contents: read
id-token: write
2022-09-02 05:40:13 +00:00
on:
workflow_call:
inputs:
ref:
required: true
description: "GitHub ref to use"
type: string
version:
required: true
description: "Version to produce"
type: string
build-all-targets:
required: false
default: true
description: "Build targets to produce, false builds only for Linux amd64."
type: boolean
ci: Don't run codegen tests for PRs We currently run all codegen tests in pkg/codegen/$lang for every PR. These tests take quite a while to run and lock up many GitHub workers for this entire duration. This change attempts to address this issue by running codegen tests only for those PRs that touch the codegen directories. The machinery to make this work is roughly as follows: - In the on-pr workflow, when we're figuring out what we're doing, we check if we've changed codegen files. We use [paths-filter] to do this. - We decide whether we want to run codegen tests based on those files, and pass that onto the test matrix generator. - The test matrix generator filters out these packages and their subpackages from the list of tests under consideration. - Everything else proceeds as normal. [paths-filter]: https://github.com/dorny/paths-filter Things to note: - The test-codegen input defaults to true. All other invocations will run with codegen tests so these will continue to run on merge. Only PRs (from on-pr.yml) set it to false. - Since the number of tests is remarkably smaller without these tests, we can significantly reduce the number of partitions we use for pkg/ unit tests. This should alleviate pressure on GitHub workers further. This is a pretty blunt approach to the problem. If we wanted to be more targeted, instead of filtering at the get-job-matrix.py level, we could instead set an environment variable and add t.Skips in {program,sdk,type}_driver if that environment variable is set. And we can still do that in the future if we decide that maintaining this list is too much. Resolves #12334
2023-03-02 22:33:45 +00:00
test-codegen:
required: false
default: true
description: "Whether to run per-language codegen tests."
type: boolean
2022-09-02 05:40:13 +00:00
lint:
required: false
default: true
description: "Whether to run lints"
type: boolean
test-version-sets:
2022-09-02 05:40:13 +00:00
required: false
default: minimum current
description: Version sets on which to run integration tests
2022-09-02 05:40:13 +00:00
type: string
integration-test-platforms:
2022-09-02 05:40:13 +00:00
required: false
default: ubuntu-latest
description: Platforms on which to run integration tests, as a space delimited list
type: string
acceptance-test-platforms:
required: false
default: windows-latest macos-latest
description: Platforms on which to run integration tests, as a space delimited list
2022-09-02 05:40:13 +00:00
type: string
enable-coverage:
description: "Collects coverage stats; requires cov-enabled builds"
default: false
2022-09-02 05:40:13 +00:00
required: false
type: boolean
secrets:
PULUMI_BOT_TOKEN:
required: true
description: "GitHub access token, required to mitigate GitHub rate limits"
PULUMI_PROD_ACCESS_TOKEN:
required: false
description: "Pulumi access token, required to run tests against the service"
CODECOV_TOKEN:
required: false
description: "CodeCov token, required to publish CodeCov coverage data"
AZURE_TENANT_ID:
required: false
description: "Azure tenant ID, required to run tests against Azure"
AZURE_CLIENT_ID:
required: false
description: "Azure client ID, required to run tests against Azure"
AZURE_CLIENT_SECRET:
required: false
description: "Azure clients secret, needs to be rotated before 2025-12-21 (see the pulumi-test user in Azure portal)"
AZURE_STORAGE_SAS_TOKEN:
required: false
description: "Azure storage SAS token, required to run tests against Azure"
GCP_SERVICE_ACCOUNT:
required: false
description: "GCP service account, required to run tests against GCP"
2022-09-02 05:40:13 +00:00
jobs:
matrix:
2022-09-02 05:40:13 +00:00
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
ref: ${{ inputs.ref }}
- name: Configure Go Cache Key
env:
CACHE_KEY: "matrix-setup"
run: echo "$CACHE_KEY" > .gocache.tmp
- name: Setup Go Caching
uses: actions/setup-go@v3 # only used by gotestsum
with:
go-version: '>=1.19.0' # decoupled from version sets, only used by gotestsum
cache: true
cache-dependency-path: |
pkg/go.sum
.gocache.tmp
- uses: actions/checkout@v3
with:
repository: dnephin/gotestsum
ref: d09768c81065b404caed0855eb3ab8f11a2a4431
path: vendor/gotestsum
- run: |
cd vendor/gotestsum
go install .
- uses: actions/cache@v3
with:
path: test-results
key: read-gotestsum-timing-${{ github.run_number }}
restore-keys: gotestsum-timing-
- name: build matrix
id: matrix
env:
BUILD_ALL_TARGETS: ${{ inputs.build-all-targets }}
ci: Don't run codegen tests for PRs We currently run all codegen tests in pkg/codegen/$lang for every PR. These tests take quite a while to run and lock up many GitHub workers for this entire duration. This change attempts to address this issue by running codegen tests only for those PRs that touch the codegen directories. The machinery to make this work is roughly as follows: - In the on-pr workflow, when we're figuring out what we're doing, we check if we've changed codegen files. We use [paths-filter] to do this. - We decide whether we want to run codegen tests based on those files, and pass that onto the test matrix generator. - The test matrix generator filters out these packages and their subpackages from the list of tests under consideration. - Everything else proceeds as normal. [paths-filter]: https://github.com/dorny/paths-filter Things to note: - The test-codegen input defaults to true. All other invocations will run with codegen tests so these will continue to run on merge. Only PRs (from on-pr.yml) set it to false. - Since the number of tests is remarkably smaller without these tests, we can significantly reduce the number of partitions we use for pkg/ unit tests. This should alleviate pressure on GitHub workers further. This is a pretty blunt approach to the problem. If we wanted to be more targeted, instead of filtering at the get-job-matrix.py level, we could instead set an environment variable and add t.Skips in {program,sdk,type}_driver if that environment variable is set. And we can still do that in the future if we decide that maintaining this list is too much. Resolves #12334
2023-03-02 22:33:45 +00:00
TEST_CODEGEN: ${{ inputs.test-codegen }}
TEST_VERSION_SETS: ${{ inputs.test-version-sets }}
INPUT_INTEGRATION_TEST_PLATFORMS: ${{ inputs.integration-test-platforms }}
INPUT_ACCEPTANCE_TEST_PLATFORMS: ${{ inputs.acceptance-test-platforms }}
run: |
echo "::group::Prime test timing data"
mkdir -p test-results
find test-results -type f -empty -print -delete || true
echo "::endgroup::"
2022-09-02 05:40:13 +00:00
2022-10-17 08:58:11 +00:00
echo "::group::Remove old test timing data"
# Timing data prior to this date is unreliable. Codegen tests modified in #11052 and
# merged Monday Oct 17 at 6PM Pacific.
find test-results -type f ! -newermt "2022-10-17T18:00-07:00" -print -delete || true
echo "::endgroup::"
echo "::group::Test matrix variables"
readarray -td' ' VERSION_SETS_TO_TEST < <(echo -n "$TEST_VERSION_SETS"); declare -p VERSION_SETS_TO_TEST;
readarray -td' ' INTEGRATION_PLATFORMS < <(echo -n "$INPUT_INTEGRATION_TEST_PLATFORMS"); declare -p INTEGRATION_PLATFORMS;
readarray -td' ' ACCEPTANCE_PLATFORMS < <(echo -n "$INPUT_ACCEPTANCE_TEST_PLATFORMS"); declare -p ACCEPTANCE_PLATFORMS;
BUILD_TARGETS='[
{ "os": "linux", "arch": "amd64", "build-platform": "ubuntu-latest" }
]'
if [ "${BUILD_ALL_TARGETS}" = "true" ]; then
BUILD_TARGETS='[
{ "os": "linux", "arch": "amd64", "build-platform": "ubuntu-latest" },
{ "os": "linux", "arch": "arm64", "build-platform": "ubuntu-latest" },
{ "os": "windows", "arch": "amd64", "build-platform": "ubuntu-latest" },
{ "os": "windows", "arch": "arm64", "build-platform": "ubuntu-latest" },
{ "os": "darwin", "arch": "amd64", "build-platform": "ubuntu-latest" },
{ "os": "darwin", "arch": "arm64", "build-platform": "ubuntu-latest" }
]'
fi
ci: Don't run codegen tests for PRs We currently run all codegen tests in pkg/codegen/$lang for every PR. These tests take quite a while to run and lock up many GitHub workers for this entire duration. This change attempts to address this issue by running codegen tests only for those PRs that touch the codegen directories. The machinery to make this work is roughly as follows: - In the on-pr workflow, when we're figuring out what we're doing, we check if we've changed codegen files. We use [paths-filter] to do this. - We decide whether we want to run codegen tests based on those files, and pass that onto the test matrix generator. - The test matrix generator filters out these packages and their subpackages from the list of tests under consideration. - Everything else proceeds as normal. [paths-filter]: https://github.com/dorny/paths-filter Things to note: - The test-codegen input defaults to true. All other invocations will run with codegen tests so these will continue to run on merge. Only PRs (from on-pr.yml) set it to false. - Since the number of tests is remarkably smaller without these tests, we can significantly reduce the number of partitions we use for pkg/ unit tests. This should alleviate pressure on GitHub workers further. This is a pretty blunt approach to the problem. If we wanted to be more targeted, instead of filtering at the get-job-matrix.py level, we could instead set an environment variable and add t.Skips in {program,sdk,type}_driver if that environment variable is set. And we can still do that in the future if we decide that maintaining this list is too much. Resolves #12334
2023-03-02 22:33:45 +00:00
CODEGEN_TESTS_FLAG=--codegen-tests
PKG_UNIT_TEST_PARTITIONS=7
if [ "${TEST_CODEGEN}" = "false" ]; then
CODEGEN_TESTS_FLAG=--no-codegen-tests
PKG_UNIT_TEST_PARTITIONS=3
fi
UNIT_TEST_MATRIX=$(
./scripts/get-job-matrix.py \
-vvv \
generate-matrix \
--kind unit-test \
ci: Don't run codegen tests for PRs We currently run all codegen tests in pkg/codegen/$lang for every PR. These tests take quite a while to run and lock up many GitHub workers for this entire duration. This change attempts to address this issue by running codegen tests only for those PRs that touch the codegen directories. The machinery to make this work is roughly as follows: - In the on-pr workflow, when we're figuring out what we're doing, we check if we've changed codegen files. We use [paths-filter] to do this. - We decide whether we want to run codegen tests based on those files, and pass that onto the test matrix generator. - The test matrix generator filters out these packages and their subpackages from the list of tests under consideration. - Everything else proceeds as normal. [paths-filter]: https://github.com/dorny/paths-filter Things to note: - The test-codegen input defaults to true. All other invocations will run with codegen tests so these will continue to run on merge. Only PRs (from on-pr.yml) set it to false. - Since the number of tests is remarkably smaller without these tests, we can significantly reduce the number of partitions we use for pkg/ unit tests. This should alleviate pressure on GitHub workers further. This is a pretty blunt approach to the problem. If we wanted to be more targeted, instead of filtering at the get-job-matrix.py level, we could instead set an environment variable and add t.Skips in {program,sdk,type}_driver if that environment variable is set. And we can still do that in the future if we decide that maintaining this list is too much. Resolves #12334
2023-03-02 22:33:45 +00:00
"$CODEGEN_TESTS_FLAG" \
--platform ubuntu-latest \
--version-set current \
2024-01-25 16:26:55 +00:00
--partition-module cmd/pulumi-test-language 1 \
ci: Don't run codegen tests for PRs We currently run all codegen tests in pkg/codegen/$lang for every PR. These tests take quite a while to run and lock up many GitHub workers for this entire duration. This change attempts to address this issue by running codegen tests only for those PRs that touch the codegen directories. The machinery to make this work is roughly as follows: - In the on-pr workflow, when we're figuring out what we're doing, we check if we've changed codegen files. We use [paths-filter] to do this. - We decide whether we want to run codegen tests based on those files, and pass that onto the test matrix generator. - The test matrix generator filters out these packages and their subpackages from the list of tests under consideration. - Everything else proceeds as normal. [paths-filter]: https://github.com/dorny/paths-filter Things to note: - The test-codegen input defaults to true. All other invocations will run with codegen tests so these will continue to run on merge. Only PRs (from on-pr.yml) set it to false. - Since the number of tests is remarkably smaller without these tests, we can significantly reduce the number of partitions we use for pkg/ unit tests. This should alleviate pressure on GitHub workers further. This is a pretty blunt approach to the problem. If we wanted to be more targeted, instead of filtering at the get-job-matrix.py level, we could instead set an environment variable and add t.Skips in {program,sdk,type}_driver if that environment variable is set. And we can still do that in the future if we decide that maintaining this list is too much. Resolves #12334
2023-03-02 22:33:45 +00:00
--partition-module pkg "$PKG_UNIT_TEST_PARTITIONS" \
--partition-module sdk 1 \
--partition-module sdk/go/pulumi-language-go 1 \
--partition-module sdk/nodejs/cmd/pulumi-language-nodejs 1 \
--partition-module sdk/python/cmd/pulumi-language-python 1 \
--partition-module tests 2
)
INTEGRATION_TEST_MATRIX=$(
./scripts/get-job-matrix.py \
-vvv \
generate-matrix \
--kind integration-test \
ci: Don't run codegen tests for PRs We currently run all codegen tests in pkg/codegen/$lang for every PR. These tests take quite a while to run and lock up many GitHub workers for this entire duration. This change attempts to address this issue by running codegen tests only for those PRs that touch the codegen directories. The machinery to make this work is roughly as follows: - In the on-pr workflow, when we're figuring out what we're doing, we check if we've changed codegen files. We use [paths-filter] to do this. - We decide whether we want to run codegen tests based on those files, and pass that onto the test matrix generator. - The test matrix generator filters out these packages and their subpackages from the list of tests under consideration. - Everything else proceeds as normal. [paths-filter]: https://github.com/dorny/paths-filter Things to note: - The test-codegen input defaults to true. All other invocations will run with codegen tests so these will continue to run on merge. Only PRs (from on-pr.yml) set it to false. - Since the number of tests is remarkably smaller without these tests, we can significantly reduce the number of partitions we use for pkg/ unit tests. This should alleviate pressure on GitHub workers further. This is a pretty blunt approach to the problem. If we wanted to be more targeted, instead of filtering at the get-job-matrix.py level, we could instead set an environment variable and add t.Skips in {program,sdk,type}_driver if that environment variable is set. And we can still do that in the future if we decide that maintaining this list is too much. Resolves #12334
2023-03-02 22:33:45 +00:00
"$CODEGEN_TESTS_FLAG" \
--platform "${INTEGRATION_PLATFORMS[@]}" \
--version-set "${VERSION_SETS_TO_TEST[@]}" \
--partition-module pkg 1 \
--partition-module sdk 1 \
--partition-module tests 2 \
--partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8
)
ACCEPTANCE_TEST_MATRIX_WIN="{}"
if [[ " ${ACCEPTANCE_PLATFORMS[*]} " =~ [[:space:]]windows-latest[[:space:]] ]]; then
ACCEPTANCE_TEST_MATRIX_WIN=$(
./scripts/get-job-matrix.py \
-vvv \
generate-matrix \
--kind acceptance-test \
"$CODEGEN_TESTS_FLAG" \
--tags all xplatform_acceptance \
--platform windows-latest \
--version-set current \
--partition-module pkg 1 \
--partition-module sdk 1 \
--partition-module tests 1 \
--partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8
)
fi
ACCEPTANCE_TEST_MATRIX_MACOS="{}"
if [[ " ${ACCEPTANCE_PLATFORMS[*]} " =~ [[:space:]]macos-latest[[:space:]] ]]; then
ACCEPTANCE_TEST_MATRIX_MACOS=$(
./scripts/get-job-matrix.py \
-vvv \
generate-matrix \
--kind acceptance-test \
"$CODEGEN_TESTS_FLAG" \
--tags all xplatform_acceptance \
--platform macos-latest \
--version-set current \
--partition-module pkg 1 \
--partition-module sdk 1 \
--partition-module tests 1 \
--partition-package github.com/pulumi/pulumi/tests/integration tests/integration 2
)
fi
echo "::endgroup::"
echo "::group::Version set variable"
VERSION_SET=$(./scripts/get-job-matrix.py \
generate-version-set \
--version-set current
)
echo "::endgroup::"
echo "::group::Unit test matrix"
echo "$UNIT_TEST_MATRIX" | yq -P '.'
echo "::endgroup::"
echo "::group::Integration test matrix"
echo "$INTEGRATION_TEST_MATRIX" | yq -P '.'
echo "::endgroup::"
echo "::group::acceptance test matrix windows"
echo "$ACCEPTANCE_TEST_MATRIX_WIN" | yq -P '.'
echo "::endgroup::"
echo "::group::acceptance test matrix macos"
echo "$ACCEPTANCE_TEST_MATRIX_MACOS" | yq -P '.'
echo "::endgroup::"
echo "::group::Version set"
echo "$VERSION_SET" | yq -P '.'
echo "::endgroup::"
echo "::group::Set outputs"
./.github/scripts/set-output unit-test-matrix "${UNIT_TEST_MATRIX}"
./.github/scripts/set-output integration-test-matrix "${INTEGRATION_TEST_MATRIX}"
./.github/scripts/set-output acceptance-test-matrix-win "${ACCEPTANCE_TEST_MATRIX_WIN}"
./.github/scripts/set-output acceptance-test-matrix-macos "${ACCEPTANCE_TEST_MATRIX_MACOS}"
./.github/scripts/set-output version-set "${VERSION_SET}"
./.github/scripts/set-output build-targets "${BUILD_TARGETS}"
echo "::endgroup::"
outputs:
unit-test-matrix: "${{ fromJson(steps.matrix.outputs.unit-test-matrix) }}"
integration-test-matrix: "${{ fromJson(steps.matrix.outputs.integration-test-matrix) }}"
acceptance-test-matrix-win: "${{ fromJson(steps.matrix.outputs.acceptance-test-matrix-win) }}"
acceptance-test-matrix-macos: "${{ fromJson(steps.matrix.outputs.acceptance-test-matrix-macos) }}"
version-set: "${{ fromJson(steps.matrix.outputs.version-set) }}"
build-targets: "${{ fromJson(steps.matrix.outputs.build-targets) }}"
lint:
name: Lint
needs: [matrix]
if: ${{ inputs.lint }}
uses: ./.github/workflows/ci-lint.yml
2022-09-02 05:40:13 +00:00
with:
ref: ${{ inputs.ref }}
version-set: ${{ needs.matrix.outputs.version-set }}
2022-09-02 05:40:13 +00:00
build-binaries:
name: build binaries
needs: [matrix]
2022-09-02 05:40:13 +00:00
strategy:
# To avoid tying up macOS runners:
# If using IDE, ignore yaml-schema error: 'Incorrect type. Expected "boolean"'
fail-fast: ${{ contains(needs.matrix.outputs.build-targets, 'macos') }}
2022-09-02 05:40:13 +00:00
matrix:
target: ${{ fromJson(needs.matrix.outputs.build-targets) }}
2022-09-02 05:40:13 +00:00
uses: ./.github/workflows/ci-build-binaries.yml
with:
ref: ${{ inputs.ref }}
version: ${{ inputs.version }}
os: ${{ matrix.target.os }}
arch: ${{ matrix.target.arch }}
build-platform: ${{ matrix.target.build-platform }}
version-set: ${{ needs.matrix.outputs.version-set }}
ci: Track code coverage **Overview** This re-enables tracking of code coverage. For Go, there are two kinds of coverage at play: unit test and integration test coverage. Unit tests follow the usual pattern of running `go test -cover -coverprofile=whatever.cov`. For integration tests, we use the new integration test profiling support [added in Go 1.20](https://go.dev/testing/coverage/). In short, the way it works is: # Build a coverage instrumented binary: go build -cover # Set GOCOVERDIR to a directory and run the integration tests # that will invoke this coverage-instrumented binary. GOCOVERDIR=$(pwd)/coverage go test ./tests # $GOCOVERDIR will now be filled with coverage data # from every invocation of the coverage-instrumented binary. # Combine it into a single coverage file: go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov # The resulting file can be uploaded to codecov as-is. The above replaces the prior, partially working hacks we had in place to get coverage-instrumented binaries with `go test -c` and hijacking the TestMain. **Notable changes** - TestMain hijacking is deleted from the Pulumi CLI. We no longer need this to build coverage-instrumented binaries. - ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH because the Pulumi binary no longer accepts a test.coverprofile flag. This information is now in the GOCOVERDIR environment variable. - We add an `enable-coverage` parameter to the `ci-build-binaries` workflow to mirror some of the other workflows. It will produce coverage-instrumented binaries if this is true. These binaries are then used by `ci-run-test` which will set `GOCOVERDIR` and merge the coverage results from it. - Coverage configuration no longer counts tests, testdata, and Protobuf-generated code against coverage. - go-wrapper.sh: Because we're no longer relying on the `go test -c` hack, this no longer excludes Windows and language providers from coverage tracking. - go-test.py and go-wrapper.sh will include pulumi-language-go and pulumi-language-nodejs in covered packages. *Other changes* - go-test.py: Fixed a bug where `args` parameters added for coverage were ignored. Note that this change DOES NOT track coverage for calls made to Pulumi packages by plugins downloaded from external sources, e.g. provider plugins. Arguably, that's out of scope of coverage trackcing for the Pulumi repository. Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
enable-coverage: ${{ inputs.enable-coverage }}
2022-09-02 05:40:13 +00:00
secrets: inherit
build-sdks:
name: Build SDKs
needs: [matrix]
2022-09-02 05:40:13 +00:00
uses: ./.github/workflows/ci-build-sdks.yml
with:
ref: ${{ inputs.ref }}
version: ${{ inputs.version }}
version-set: ${{ needs.matrix.outputs.version-set }}
secrets: inherit
# Tests that can run concurrently with builds.
unit-test:
# By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
# appearing in the rendered title of the job name: It changes this:
# CI / Unit Test (cd sdk/dotnet && make dotnet_test, cd sdk/dotnet && make dotnet_test, macos-11, mi... / sdk/dotnet dotnet_test on macos-11/current
# (See: https://github.com/pulumi/pulumi/runs/8241055084?check_suite_focus=true#logs)
# To this:
# CI / Unit Test / sdk/dotnet dotnet_test on macos-11/current
name: Unit Test${{ matrix.platform && '' }}
needs: [matrix, lint]
if: ${{ needs.matrix.outputs.unit-test-matrix != '{}' }}
strategy:
fail-fast: ${{ contains(needs.matrix.outputs.unit-test-matrix, 'macos') }}
matrix: ${{ fromJson(needs.matrix.outputs.unit-test-matrix) }}
uses: ./.github/workflows/ci-run-test.yml
with:
ref: ${{ inputs.ref }}
version: ${{ inputs.version }}
platform: ${{ matrix.platform }}
test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
test-command: ${{ matrix.test-suite.command }}
is-integration-test: false
ci: Track code coverage **Overview** This re-enables tracking of code coverage. For Go, there are two kinds of coverage at play: unit test and integration test coverage. Unit tests follow the usual pattern of running `go test -cover -coverprofile=whatever.cov`. For integration tests, we use the new integration test profiling support [added in Go 1.20](https://go.dev/testing/coverage/). In short, the way it works is: # Build a coverage instrumented binary: go build -cover # Set GOCOVERDIR to a directory and run the integration tests # that will invoke this coverage-instrumented binary. GOCOVERDIR=$(pwd)/coverage go test ./tests # $GOCOVERDIR will now be filled with coverage data # from every invocation of the coverage-instrumented binary. # Combine it into a single coverage file: go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov # The resulting file can be uploaded to codecov as-is. The above replaces the prior, partially working hacks we had in place to get coverage-instrumented binaries with `go test -c` and hijacking the TestMain. **Notable changes** - TestMain hijacking is deleted from the Pulumi CLI. We no longer need this to build coverage-instrumented binaries. - ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH because the Pulumi binary no longer accepts a test.coverprofile flag. This information is now in the GOCOVERDIR environment variable. - We add an `enable-coverage` parameter to the `ci-build-binaries` workflow to mirror some of the other workflows. It will produce coverage-instrumented binaries if this is true. These binaries are then used by `ci-run-test` which will set `GOCOVERDIR` and merge the coverage results from it. - Coverage configuration no longer counts tests, testdata, and Protobuf-generated code against coverage. - go-wrapper.sh: Because we're no longer relying on the `go test -c` hack, this no longer excludes Windows and language providers from coverage tracking. - go-test.py and go-wrapper.sh will include pulumi-language-go and pulumi-language-nodejs in covered packages. *Other changes* - go-test.py: Fixed a bug where `args` parameters added for coverage were ignored. Note that this change DOES NOT track coverage for calls made to Pulumi packages by plugins downloaded from external sources, e.g. provider plugins. Arguably, that's out of scope of coverage trackcing for the Pulumi repository. Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
enable-coverage: ${{ inputs.enable-coverage }}
# require-build: false # TODO, remove ${{ matrix.require-build || false }}
version-set: ${{ toJson(matrix.version-set) }}
2022-09-02 05:40:13 +00:00
secrets: inherit
# Tests that depend on builds
2022-09-02 05:40:13 +00:00
integration-test:
# By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
# appearing in the rendered title of the job name. See: unit test.
name: Integration Test${{ matrix.platform && '' }}
needs: [matrix, build-binaries, build-sdks, lint]
if: ${{ needs.matrix.outputs.integration-test-matrix != '{}' }}
2022-09-02 05:40:13 +00:00
strategy:
fail-fast: ${{ contains(needs.matrix.outputs.integration-test-matrix, 'macos') }}
matrix: ${{ fromJson(needs.matrix.outputs.integration-test-matrix) }}
uses: ./.github/workflows/ci-run-test.yml
2022-09-02 05:40:13 +00:00
with:
ref: ${{ inputs.ref }}
version: ${{ inputs.version }}
platform: ${{ matrix.platform }}
test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
test-command: ${{ matrix.test-suite.command }}
is-integration-test: true
ci: Track code coverage **Overview** This re-enables tracking of code coverage. For Go, there are two kinds of coverage at play: unit test and integration test coverage. Unit tests follow the usual pattern of running `go test -cover -coverprofile=whatever.cov`. For integration tests, we use the new integration test profiling support [added in Go 1.20](https://go.dev/testing/coverage/). In short, the way it works is: # Build a coverage instrumented binary: go build -cover # Set GOCOVERDIR to a directory and run the integration tests # that will invoke this coverage-instrumented binary. GOCOVERDIR=$(pwd)/coverage go test ./tests # $GOCOVERDIR will now be filled with coverage data # from every invocation of the coverage-instrumented binary. # Combine it into a single coverage file: go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov # The resulting file can be uploaded to codecov as-is. The above replaces the prior, partially working hacks we had in place to get coverage-instrumented binaries with `go test -c` and hijacking the TestMain. **Notable changes** - TestMain hijacking is deleted from the Pulumi CLI. We no longer need this to build coverage-instrumented binaries. - ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH because the Pulumi binary no longer accepts a test.coverprofile flag. This information is now in the GOCOVERDIR environment variable. - We add an `enable-coverage` parameter to the `ci-build-binaries` workflow to mirror some of the other workflows. It will produce coverage-instrumented binaries if this is true. These binaries are then used by `ci-run-test` which will set `GOCOVERDIR` and merge the coverage results from it. - Coverage configuration no longer counts tests, testdata, and Protobuf-generated code against coverage. - go-wrapper.sh: Because we're no longer relying on the `go test -c` hack, this no longer excludes Windows and language providers from coverage tracking. - go-test.py and go-wrapper.sh will include pulumi-language-go and pulumi-language-nodejs in covered packages. *Other changes* - go-test.py: Fixed a bug where `args` parameters added for coverage were ignored. Note that this change DOES NOT track coverage for calls made to Pulumi packages by plugins downloaded from external sources, e.g. provider plugins. Arguably, that's out of scope of coverage trackcing for the Pulumi repository. Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
enable-coverage: ${{ inputs.enable-coverage }}
# require-build: false # TODO, remove ${{ matrix.require-build || false }}
version-set: ${{ toJson(matrix.version-set) }}
2022-09-02 05:40:13 +00:00
secrets: inherit
# Tests that depend on builds, but a smaller subset against Windows platform.
acceptance-test-win:
# By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
# appearing in the rendered title of the job name. See: unit test.
name: Acceptance Test${{ matrix.platform && '' }}
needs: [matrix, build-binaries, build-sdks]
if: ${{ needs.matrix.outputs.acceptance-test-matrix-win != '{}' }}
# alow jobs to fail if the platform contains windows
strategy:
matrix: ${{ fromJson(needs.matrix.outputs.acceptance-test-matrix-win) }}
uses: ./.github/workflows/ci-run-test.yml
with:
ref: ${{ inputs.ref }}
version: ${{ inputs.version }}
platform: ${{ matrix.platform }}
test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
test-command: ${{ matrix.test-suite.command }}
is-integration-test: true
enable-coverage: ${{ inputs.enable-coverage }}
# require-build: false # TODO, remove ${{ matrix.require-build || false }}
version-set: ${{ toJson(matrix.version-set) }}
secrets: inherit
# Tests that depend on builds, but a smaller subset against MacOS platform.
acceptance-test-macos:
# By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
# appearing in the rendered title of the job name. See: unit test.
2023-01-30 20:39:04 +00:00
name: Acceptance Test${{ matrix.platform && '' }}
needs: [matrix, build-binaries, build-sdks]
if: ${{ needs.matrix.outputs.acceptance-test-matrix-macos != '{}' }}
# alow jobs to fail if the platform contains windows
strategy:
fail-fast: true
matrix: ${{ fromJson(needs.matrix.outputs.acceptance-test-matrix-macos) }}
uses: ./.github/workflows/ci-run-test.yml
with:
ref: ${{ inputs.ref }}
version: ${{ inputs.version }}
platform: ${{ matrix.platform }}
test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
test-command: ${{ matrix.test-suite.command }}
is-integration-test: true
ci: Track code coverage **Overview** This re-enables tracking of code coverage. For Go, there are two kinds of coverage at play: unit test and integration test coverage. Unit tests follow the usual pattern of running `go test -cover -coverprofile=whatever.cov`. For integration tests, we use the new integration test profiling support [added in Go 1.20](https://go.dev/testing/coverage/). In short, the way it works is: # Build a coverage instrumented binary: go build -cover # Set GOCOVERDIR to a directory and run the integration tests # that will invoke this coverage-instrumented binary. GOCOVERDIR=$(pwd)/coverage go test ./tests # $GOCOVERDIR will now be filled with coverage data # from every invocation of the coverage-instrumented binary. # Combine it into a single coverage file: go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov # The resulting file can be uploaded to codecov as-is. The above replaces the prior, partially working hacks we had in place to get coverage-instrumented binaries with `go test -c` and hijacking the TestMain. **Notable changes** - TestMain hijacking is deleted from the Pulumi CLI. We no longer need this to build coverage-instrumented binaries. - ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH because the Pulumi binary no longer accepts a test.coverprofile flag. This information is now in the GOCOVERDIR environment variable. - We add an `enable-coverage` parameter to the `ci-build-binaries` workflow to mirror some of the other workflows. It will produce coverage-instrumented binaries if this is true. These binaries are then used by `ci-run-test` which will set `GOCOVERDIR` and merge the coverage results from it. - Coverage configuration no longer counts tests, testdata, and Protobuf-generated code against coverage. - go-wrapper.sh: Because we're no longer relying on the `go test -c` hack, this no longer excludes Windows and language providers from coverage tracking. - go-test.py and go-wrapper.sh will include pulumi-language-go and pulumi-language-nodejs in covered packages. *Other changes* - go-test.py: Fixed a bug where `args` parameters added for coverage were ignored. Note that this change DOES NOT track coverage for calls made to Pulumi packages by plugins downloaded from external sources, e.g. provider plugins. Arguably, that's out of scope of coverage trackcing for the Pulumi repository. Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
enable-coverage: ${{ inputs.enable-coverage }}
# require-build: false # TODO, remove ${{ matrix.require-build || false }}
version-set: ${{ toJson(matrix.version-set) }}
secrets: inherit
test-collect-reports:
needs: [unit-test, integration-test, acceptance-test-win, acceptance-test-macos]
if: ${{ always() }}
runs-on: ubuntu-latest
steps:
- uses: actions/cache@v3
with:
path: test-results
key: gotestsum-timing-${{ github.run_number }}
restore-keys: gotestsum-timing-
- uses: actions/download-artifact@v3
continue-on-error: true
with:
name: gotestsum-test-results
path: test-results
- name: List and clean up test results
continue-on-error: true
run: |
ls -lhR test-results
find test-results -mindepth 1 -name '*.json' -mtime +7 -delete
test-collect-coverage:
needs: [unit-test, integration-test, acceptance-test-win, acceptance-test-macos]
if: ${{ inputs.enable-coverage }}
runs-on: ubuntu-latest
steps:
- name: Retrieve code coverage statuses
uses: actions/download-artifact@v3
with:
path: status
# Check that there are no incomplete tests.
- name: Check tests completed successfully.
run: |
if test -z "$(grep -rl 'incomplete' ./status)"; then
echo "Tests OK!"
else
echo "Test Failure.. skipping CodeCov upload."
exit 1
fi
# Checkout repository to upload coverage results.
- uses: actions/checkout@v3
with:
ref: ${{ inputs.ref }}
- name: Retrieve code coverage reports
uses: actions/download-artifact@v3
with:
path: coverage
- name: Upload code coverage
uses: codecov/codecov-action@v3
with:
directory: coverage/
files: "*,!.gitkeep"
fail_ci_if_error: false
verbose: true
token: ${{ secrets.CODECOV_TOKEN }}
build-release-binaries:
# This overwrites the previously built testing binaries with versions without coverage.
name: Rebuild binaries
needs: [matrix, unit-test, integration-test, acceptance-test-win, acceptance-test-macos]
if: ${{ inputs.enable-coverage }}
strategy:
# To avoid tying up macOS runners:
# If using IDE, ignore yaml-schema error: 'Incorrect type. Expected "boolean"'
fail-fast: ${{ contains(needs.matrix.outputs.build-targets, 'macos') }}
matrix:
target: ${{ fromJson(needs.matrix.outputs.build-targets) }}
uses: ./.github/workflows/ci-build-binaries.yml
with:
ref: ${{ inputs.ref }}
version: ${{ inputs.version }}
os: ${{ matrix.target.os }}
arch: ${{ matrix.target.arch }}
build-platform: ${{ matrix.target.build-platform }}
version-set: ${{ needs.matrix.outputs.version-set }}
enable-coverage: false
secrets: inherit