name: CI permissions: contents: read id-token: write on: workflow_call: inputs: ref: required: true description: "GitHub ref to use" type: string version: required: true description: "Version to produce" type: string build-all-targets: required: false default: true description: "Build targets to produce, false builds only for Linux amd64." type: boolean test-codegen: required: false default: true description: "Whether to run per-language codegen tests." type: boolean lint: required: false default: true description: "Whether to run lints" type: boolean test-version-sets: required: false default: minimum current description: Version sets on which to run integration tests type: string integration-test-platforms: required: false default: ubuntu-latest description: Platforms on which to run integration tests, as a space delimited list type: string acceptance-test-platforms: required: false default: windows-latest macos-latest description: Platforms on which to run integration tests, as a space delimited list type: string enable-coverage: description: "Collects coverage stats; requires cov-enabled builds" default: false required: false type: boolean fail-fast: required: false default: false description: "Fail all workflows whenever one of them fails" type: boolean test-retries: required: false default: 0 description: "Retry tests n times if there are failures" type: number secrets: PULUMI_BOT_TOKEN: required: true description: "GitHub access token, required to mitigate GitHub rate limits" PULUMI_PROD_ACCESS_TOKEN: required: false description: "Pulumi access token, required to run tests against the service" CODECOV_TOKEN: required: false description: "CodeCov token, required to publish CodeCov coverage data" AZURE_TENANT_ID: required: false description: "Azure tenant ID, required to run tests against Azure" AZURE_CLIENT_ID: required: false description: "Azure client ID, required to run tests against Azure" AZURE_CLIENT_SECRET: required: false description: "Azure clients secret, needs to be rotated before 2025-12-21 (see the pulumi-test user in Azure portal)" AZURE_STORAGE_SAS_TOKEN: required: false description: "Azure storage SAS token, required to run tests against Azure" GCP_SERVICE_ACCOUNT: required: false description: "GCP service account, required to run tests against GCP" jobs: matrix: runs-on: ubuntu-latest strategy: fail-fast: ${{ inputs.fail-fast }} steps: - uses: actions/checkout@v4 with: ref: ${{ inputs.ref }} - name: Configure Go Cache Key env: CACHE_KEY: "matrix-setup" run: echo "$CACHE_KEY" > .gocache.tmp - name: Setup Go Caching uses: actions/setup-go@v5 # only used by gotestsum with: go-version: '>=1.19.0' # decoupled from version sets, only used by gotestsum cache: true cache-dependency-path: | pkg/go.sum .gocache.tmp - uses: actions/checkout@v4 with: repository: dnephin/gotestsum ref: d09768c81065b404caed0855eb3ab8f11a2a4431 path: vendor/gotestsum - run: | cd vendor/gotestsum go install . - uses: actions/cache@v4 with: path: test-results key: read-gotestsum-timing-${{ github.run_number }} restore-keys: gotestsum-timing- - name: build matrix id: matrix env: BUILD_ALL_TARGETS: ${{ inputs.build-all-targets }} TEST_CODEGEN: ${{ inputs.test-codegen }} TEST_VERSION_SETS: ${{ inputs.test-version-sets }} INPUT_INTEGRATION_TEST_PLATFORMS: ${{ inputs.integration-test-platforms }} INPUT_ACCEPTANCE_TEST_PLATFORMS: ${{ inputs.acceptance-test-platforms }} run: | echo "::group::Prime test timing data" mkdir -p test-results find test-results -type f -empty -print -delete || true echo "::endgroup::" echo "::group::Remove old test timing data" # Timing data prior to this date is unreliable. Codegen tests modified in #11052 and # merged Monday Oct 17 at 6PM Pacific. find test-results -type f ! -newermt "2022-10-17T18:00-07:00" -print -delete || true echo "::endgroup::" echo "::group::Test matrix variables" readarray -td' ' VERSION_SETS_TO_TEST < <(echo -n "$TEST_VERSION_SETS"); declare -p VERSION_SETS_TO_TEST; readarray -td' ' INTEGRATION_PLATFORMS < <(echo -n "$INPUT_INTEGRATION_TEST_PLATFORMS"); declare -p INTEGRATION_PLATFORMS; readarray -td' ' ACCEPTANCE_PLATFORMS < <(echo -n "$INPUT_ACCEPTANCE_TEST_PLATFORMS"); declare -p ACCEPTANCE_PLATFORMS; BUILD_TARGETS='[ { "os": "linux", "arch": "amd64", "build-platform": "ubuntu-latest" } ]' if [ "${BUILD_ALL_TARGETS}" = "true" ]; then BUILD_TARGETS='[ { "os": "linux", "arch": "amd64", "build-platform": "ubuntu-latest" }, { "os": "linux", "arch": "arm64", "build-platform": "ubuntu-latest" }, { "os": "windows", "arch": "amd64", "build-platform": "ubuntu-latest" }, { "os": "windows", "arch": "arm64", "build-platform": "ubuntu-latest" }, { "os": "darwin", "arch": "amd64", "build-platform": "ubuntu-latest" }, { "os": "darwin", "arch": "arm64", "build-platform": "ubuntu-latest" } ]' fi CODEGEN_TESTS_FLAG=--codegen-tests PKG_UNIT_TEST_PARTITIONS=7 if [ "${TEST_CODEGEN}" = "false" ]; then CODEGEN_TESTS_FLAG=--no-codegen-tests PKG_UNIT_TEST_PARTITIONS=3 fi UNIT_TEST_MATRIX=$( ./scripts/get-job-matrix.py \ -vvv \ generate-matrix \ --kind unit-test \ "$CODEGEN_TESTS_FLAG" \ --platform ubuntu-latest \ --version-set current \ --partition-module cmd/pulumi-test-language 1 \ --partition-module pkg "$PKG_UNIT_TEST_PARTITIONS" \ --partition-module sdk 1 \ --partition-module sdk/go/pulumi-language-go 1 \ --partition-module sdk/nodejs/cmd/pulumi-language-nodejs 1 \ --partition-module sdk/python/cmd/pulumi-language-python 1 \ --partition-module tests 2 ) INTEGRATION_TEST_MATRIX=$( ./scripts/get-job-matrix.py \ -vvv \ generate-matrix \ --kind integration-test \ "$CODEGEN_TESTS_FLAG" \ --platform "${INTEGRATION_PLATFORMS[@]}" \ --version-set "${VERSION_SETS_TO_TEST[@]}" \ --partition-module pkg 1 \ --partition-module sdk 1 \ --partition-module tests 2 \ --partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8 ) ACCEPTANCE_TEST_MATRIX_WIN="{}" if [[ " ${ACCEPTANCE_PLATFORMS[*]} " =~ [[:space:]]windows-latest[[:space:]] ]]; then ACCEPTANCE_TEST_MATRIX_WIN=$( ./scripts/get-job-matrix.py \ -vvv \ generate-matrix \ --kind acceptance-test \ "$CODEGEN_TESTS_FLAG" \ --tags all xplatform_acceptance \ --platform windows-latest \ --version-set current \ --partition-module pkg 1 \ --partition-module sdk 1 \ --partition-module tests 2 \ --partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8 ) fi ACCEPTANCE_TEST_MATRIX_MACOS="{}" if [[ " ${ACCEPTANCE_PLATFORMS[*]} " =~ [[:space:]]macos-latest[[:space:]] ]]; then ACCEPTANCE_TEST_MATRIX_MACOS=$( ./scripts/get-job-matrix.py \ -vvv \ generate-matrix \ --kind acceptance-test \ "$CODEGEN_TESTS_FLAG" \ --tags all xplatform_acceptance \ --platform macos-latest \ --version-set current \ --partition-module pkg 1 \ --partition-module sdk 1 \ --partition-module tests 1 \ --partition-package github.com/pulumi/pulumi/tests/integration tests/integration 2 ) fi echo "::endgroup::" echo "::group::Version set variable" VERSION_SET=$(./scripts/get-job-matrix.py \ generate-version-set \ --version-set current ) echo "::endgroup::" echo "::group::Unit test matrix" echo "$UNIT_TEST_MATRIX" | yq -P '.' echo "::endgroup::" echo "::group::Integration test matrix" echo "$INTEGRATION_TEST_MATRIX" | yq -P '.' echo "::endgroup::" echo "::group::acceptance test matrix windows" echo "$ACCEPTANCE_TEST_MATRIX_WIN" | yq -P '.' echo "::endgroup::" echo "::group::acceptance test matrix macos" echo "$ACCEPTANCE_TEST_MATRIX_MACOS" | yq -P '.' echo "::endgroup::" echo "::group::Version set" echo "$VERSION_SET" | yq -P '.' echo "::endgroup::" echo "::group::Set outputs" ./.github/scripts/set-output unit-test-matrix "${UNIT_TEST_MATRIX}" ./.github/scripts/set-output integration-test-matrix "${INTEGRATION_TEST_MATRIX}" ./.github/scripts/set-output acceptance-test-matrix-win "${ACCEPTANCE_TEST_MATRIX_WIN}" ./.github/scripts/set-output acceptance-test-matrix-macos "${ACCEPTANCE_TEST_MATRIX_MACOS}" ./.github/scripts/set-output version-set "${VERSION_SET}" ./.github/scripts/set-output build-targets "${BUILD_TARGETS}" echo "::endgroup::" outputs: unit-test-matrix: "${{ fromJson(steps.matrix.outputs.unit-test-matrix) }}" integration-test-matrix: "${{ fromJson(steps.matrix.outputs.integration-test-matrix) }}" acceptance-test-matrix-win: "${{ fromJson(steps.matrix.outputs.acceptance-test-matrix-win) }}" acceptance-test-matrix-macos: "${{ fromJson(steps.matrix.outputs.acceptance-test-matrix-macos) }}" version-set: "${{ fromJson(steps.matrix.outputs.version-set) }}" build-targets: "${{ fromJson(steps.matrix.outputs.build-targets) }}" lint: name: Lint needs: [matrix] if: ${{ inputs.lint }} uses: ./.github/workflows/ci-lint.yml strategy: fail-fast: ${{ inputs.fail-fast }} with: ref: ${{ inputs.ref }} version-set: ${{ needs.matrix.outputs.version-set }} build-binaries: name: build binaries needs: [matrix] strategy: fail-fast: ${{ inputs.fail-fast }} matrix: target: ${{ fromJson(needs.matrix.outputs.build-targets) }} uses: ./.github/workflows/ci-build-binaries.yml with: ref: ${{ inputs.ref }} version: ${{ inputs.version }} os: ${{ matrix.target.os }} arch: ${{ matrix.target.arch }} build-platform: ${{ matrix.target.build-platform }} version-set: ${{ needs.matrix.outputs.version-set }} enable-coverage: ${{ inputs.enable-coverage }} # Windows and Linux need CGO and cross compile support to support -race. So for now only enable it for darwin and amd64 linux. enable-race-detection: ${{ matrix.target.os == 'darwin' || (matrix.target.os == 'linux' && matrix.target.arch == 'amd64') }} secrets: inherit build-display-wasm-module: name: build display WASM module needs: [matrix] uses: ./.github/workflows/ci-build-binaries.yml strategy: fail-fast: ${{ inputs.fail-fast }} with: ref: ${{ inputs.ref }} version: ${{ inputs.version }} os: js arch: wasm build-platform: "ubuntu-latest" version-set: ${{ needs.matrix.outputs.version-set }} secrets: inherit build-sdks: name: Build SDKs needs: [matrix] uses: ./.github/workflows/ci-build-sdks.yml strategy: fail-fast: ${{ inputs.fail-fast }} with: ref: ${{ inputs.ref }} version: ${{ inputs.version }} version-set: ${{ needs.matrix.outputs.version-set }} secrets: inherit # Tests that can run concurrently with builds. unit-test: # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from # appearing in the rendered title of the job name: It changes this: # CI / Unit Test (cd sdk/dotnet && make dotnet_test, cd sdk/dotnet && make dotnet_test, macos-11, mi... / sdk/dotnet dotnet_test on macos-11/current # (See: https://github.com/pulumi/pulumi/runs/8241055084?check_suite_focus=true#logs) # To this: # CI / Unit Test / sdk/dotnet dotnet_test on macos-11/current name: Unit Test${{ matrix.platform && '' }} needs: [matrix, lint] if: ${{ needs.matrix.outputs.unit-test-matrix != '{}' }} strategy: fail-fast: ${{ inputs.fail-fast }} matrix: ${{ fromJson(needs.matrix.outputs.unit-test-matrix) }} uses: ./.github/workflows/ci-run-test.yml with: ref: ${{ inputs.ref }} version: ${{ inputs.version }} platform: ${{ matrix.platform }} test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }} test-command: ${{ matrix.test-suite.command }} is-integration-test: false enable-coverage: ${{ inputs.enable-coverage }} test-retries: ${{ inputs.test-retries }} # require-build: false # TODO, remove ${{ matrix.require-build || false }} version-set: ${{ toJson(matrix.version-set) }} secrets: inherit # Tests that depend on builds integration-test: # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from # appearing in the rendered title of the job name. See: unit test. name: Integration Test${{ matrix.platform && '' }} needs: [matrix, build-binaries, build-sdks, lint] if: ${{ needs.matrix.outputs.integration-test-matrix != '{}' }} strategy: fail-fast: ${{ inputs.fail-fast }} matrix: ${{ fromJson(needs.matrix.outputs.integration-test-matrix) }} uses: ./.github/workflows/ci-run-test.yml with: ref: ${{ inputs.ref }} version: ${{ inputs.version }} platform: ${{ matrix.platform }} test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }} test-command: ${{ matrix.test-suite.command }} is-integration-test: true enable-coverage: ${{ inputs.enable-coverage }} test-retries: ${{ inputs.test-retries }} # require-build: false # TODO, remove ${{ matrix.require-build || false }} version-set: ${{ toJson(matrix.version-set) }} secrets: inherit # Tests that depend on builds, but a smaller subset against Windows platform. acceptance-test-win: # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from # appearing in the rendered title of the job name. See: unit test. name: Acceptance Test${{ matrix.platform && '' }} needs: [matrix, build-binaries, build-sdks] if: ${{ needs.matrix.outputs.acceptance-test-matrix-win != '{}' }} # alow jobs to fail if the platform contains windows strategy: matrix: ${{ fromJson(needs.matrix.outputs.acceptance-test-matrix-win) }} uses: ./.github/workflows/ci-run-test.yml with: ref: ${{ inputs.ref }} version: ${{ inputs.version }} platform: ${{ matrix.platform }} test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }} test-command: ${{ matrix.test-suite.command }} is-integration-test: true enable-coverage: ${{ inputs.enable-coverage }} test-retries: ${{ inputs.test-retries }} # require-build: false # TODO, remove ${{ matrix.require-build || false }} version-set: ${{ toJson(matrix.version-set) }} secrets: inherit # Tests that depend on builds, but a smaller subset against MacOS platform. acceptance-test-macos: # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from # appearing in the rendered title of the job name. See: unit test. name: Acceptance Test${{ matrix.platform && '' }} needs: [matrix, build-binaries, build-sdks] if: ${{ needs.matrix.outputs.acceptance-test-matrix-macos != '{}' }} # alow jobs to fail if the platform contains windows strategy: fail-fast: ${{ inputs.fail-fast }} matrix: ${{ fromJson(needs.matrix.outputs.acceptance-test-matrix-macos) }} uses: ./.github/workflows/ci-run-test.yml with: ref: ${{ inputs.ref }} version: ${{ inputs.version }} platform: ${{ matrix.platform }} test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }} test-command: ${{ matrix.test-suite.command }} is-integration-test: true enable-coverage: ${{ inputs.enable-coverage }} test-retries: ${{ inputs.test-retries }} # require-build: false # TODO, remove ${{ matrix.require-build || false }} version-set: ${{ toJson(matrix.version-set) }} secrets: inherit test-collect-reports: needs: [unit-test, integration-test, acceptance-test-win, acceptance-test-macos] if: ${{ always() }} runs-on: ubuntu-latest steps: - uses: actions/cache@v4 with: path: test-results key: gotestsum-timing-${{ github.run_number }} restore-keys: gotestsum-timing- - uses: actions/download-artifact@v4 continue-on-error: true with: pattern: gotestsum-test-results-* merge-multiple: true path: test-results - name: List and clean up test results continue-on-error: true run: | ls -lhR test-results find test-results -mindepth 1 -name '*.json' -mtime +7 -delete test-collect-coverage: needs: [unit-test, integration-test, acceptance-test-win, acceptance-test-macos] if: ${{ inputs.enable-coverage }} runs-on: ubuntu-latest steps: # Check that there are no failed tests. - name: Check tests completed successfully. run: | JSON='${{ toJson(needs) }}' if test -z "$(echo $JSON | jq -r '.[] | .result' | grep -v 'success')"; then echo "Tests OK!" else echo "Test Failure.. skipping CodeCov upload." exit 1 fi # Checkout repository to upload coverage results. - uses: actions/checkout@v4 with: ref: ${{ inputs.ref }} - name: Retrieve code coverage reports uses: actions/download-artifact@v4 with: pattern: coverage-* merge-multiple: true path: coverage - name: Upload code coverage uses: codecov/codecov-action@v4 with: directory: coverage/ files: "*,!.gitkeep" fail_ci_if_error: false verbose: true token: ${{ secrets.CODECOV_TOKEN }} build-release-binaries: # This overwrites the previously built testing binaries with versions without coverage. name: Rebuild binaries needs: [matrix, unit-test, integration-test, acceptance-test-win, acceptance-test-macos] if: ${{ inputs.enable-coverage }} strategy: fail-fast: ${{ inputs.fail-fast }} matrix: target: ${{ fromJson(needs.matrix.outputs.build-targets) }} uses: ./.github/workflows/ci-build-binaries.yml with: ref: ${{ inputs.ref }} version: ${{ inputs.version }} os: ${{ matrix.target.os }} arch: ${{ matrix.target.arch }} build-platform: ${{ matrix.target.build-platform }} version-set: ${{ needs.matrix.outputs.version-set }} enable-coverage: false secrets: inherit