pulumi/.github/workflows/ci.yml

name: CI

permissions:
  contents: read
  id-token: write

on:
  workflow_call:
    inputs:
      ref:
        required: true
        description: "GitHub ref to use"
        type: string
      version:
        required: true
        description: "Version to produce"
        type: string
      test-codegen:
        required: false
        default: true
        description: "Whether to run per-language codegen tests."
        type: boolean
      lint:
        required: false
        default: true
        description: "Whether to run lints"
        type: boolean
      test-version-sets:
        required: false
        default: minimum current
        description: Version sets on which to run integration tests
        type: string
      integration-test-platforms:
        required: false
        default: ubuntu-latest
        description: Platforms on which to run integration tests, as a space delimited list
        type: string
      acceptance-test-platforms:
        required: false
        default: macos-latest
        description: Platforms on which to run integration tests, as a space delimited list
        type: string
      enable-coverage:
        description: "Collects coverage stats; requires cov-enabled builds"
        default: false
        required: false
        type: boolean
      fail-fast:
        required: false
        default: false
        description: "Fail all workflows whenever one of them fails"
        type: boolean
      test-retries:
        required: false
        default: 0
        description: "Retry tests n times if there are failures"
        type: number
    secrets:
      PULUMI_PROD_ACCESS_TOKEN:
        required: false
        description: "Pulumi access token, required to run tests against the service"
      CODECOV_TOKEN:
        required: false
        description: "CodeCov token, required to publish CodeCov coverage data"
      AZURE_TENANT_ID:
        required: false
        description: "Azure tenant ID, required to run tests against Azure"
      AZURE_CLIENT_ID:
        required: false
        description: "Azure client ID, required to run tests against Azure"
      AZURE_CLIENT_SECRET:
        required: false
        description: "Azure clients secret, needs to be rotated before 2025-12-21 (see the pulumi-test user in Azure portal)"
      AZURE_STORAGE_SAS_TOKEN:
        required: false
        description: "Azure storage SAS token, required to run tests against Azure"
      GCP_SERVICE_ACCOUNT:
        required: false
        description: "GCP service account, required to run tests against GCP"

jobs:
  matrix:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
      - name: Configure Go Cache Key
        env:
          CACHE_KEY: "matrix-setup"
        run: echo "$CACHE_KEY" > .gocache.tmp
      - name: Setup Go Caching
        uses: actions/setup-go@v5 # only used by gotestsum
        with:
          go-version: '>=1.19.0' # decoupled from version sets, only used by gotestsum
          cache: true
          cache-dependency-path: |
            pkg/go.sum
            .gocache.tmp            
      - uses: actions/checkout@v4
        with:
          repository: dnephin/gotestsum
          ref: d09768c81065b404caed0855eb3ab8f11a2a4431
          path: vendor/gotestsum
      - run: |
          cd vendor/gotestsum
          go install .          
      - uses: actions/cache@v4
        with:
          path: test-results
          key: read-gotestsum-timing-${{ github.run_number }}
          restore-keys: gotestsum-timing-
      - name: build matrix
        id: matrix
        env:
          TEST_CODEGEN: ${{ inputs.test-codegen }}
          TEST_VERSION_SETS: ${{ inputs.test-version-sets }}
          INPUT_INTEGRATION_TEST_PLATFORMS: ${{ inputs.integration-test-platforms }}
          INPUT_ACCEPTANCE_TEST_PLATFORMS: ${{ inputs.acceptance-test-platforms }}
        run: |
          echo "::group::Prime test timing data"
          mkdir -p test-results
          find test-results -type f -empty -print -delete || true
          echo "::endgroup::"

          echo "::group::Remove old test timing data"
          # Timing data prior to this date is unreliable. Codegen tests modified in #11052 and
          # merged Monday Oct 17 at 6PM Pacific.
          find test-results -type f ! -newermt "2022-10-17T18:00-07:00" -print -delete || true
          echo "::endgroup::"

          echo "::group::Test matrix variables"
          readarray -td' ' VERSION_SETS_TO_TEST < <(echo -n "$TEST_VERSION_SETS"); declare -p VERSION_SETS_TO_TEST;
          readarray -td' ' INTEGRATION_PLATFORMS < <(echo -n "$INPUT_INTEGRATION_TEST_PLATFORMS"); declare -p INTEGRATION_PLATFORMS;
          readarray -td' ' ACCEPTANCE_PLATFORMS < <(echo -n "$INPUT_ACCEPTANCE_TEST_PLATFORMS"); declare -p ACCEPTANCE_PLATFORMS;
          BUILD_TARGETS='[
              { "os": "linux",   "arch": "amd64", "build-platform": "ubuntu-latest" },
              { "os": "linux",   "arch": "arm64", "build-platform": "ubuntu-latest" },
              { "os": "windows", "arch": "amd64", "build-platform": "ubuntu-latest" },
              { "os": "windows", "arch": "arm64", "build-platform": "ubuntu-latest" },
              { "os": "darwin",  "arch": "amd64", "build-platform": "ubuntu-latest" },
              { "os": "darwin",  "arch": "arm64", "build-platform": "ubuntu-latest" }
          ]'

          CODEGEN_TESTS_FLAG=--codegen-tests
          PKG_UNIT_TEST_PARTITIONS=7
          if [ "${TEST_CODEGEN}" = "false" ]; then
            CODEGEN_TESTS_FLAG=--no-codegen-tests
            PKG_UNIT_TEST_PARTITIONS=3
          fi

          UNIT_TEST_MATRIX=$(
            ./scripts/get-job-matrix.py \
            -vvv \
            generate-matrix \
            --kind unit-test \
            "$CODEGEN_TESTS_FLAG" \
            --platform ubuntu-latest \
            --version-set current \
            --partition-module cmd/pulumi-test-language 1 \
            --partition-module pkg "$PKG_UNIT_TEST_PARTITIONS" \
            --partition-module sdk 1 \
            --partition-module sdk/go/pulumi-language-go 1 \
            --partition-module sdk/nodejs/cmd/pulumi-language-nodejs 1 \
            --partition-module sdk/python/cmd/pulumi-language-python 1 \
            --partition-module tests 2
          )

          INTEGRATION_TEST_MATRIX=$(
            ./scripts/get-job-matrix.py \
            -vvv \
            generate-matrix \
            --kind integration-test \
            "$CODEGEN_TESTS_FLAG" \
            --platform "${INTEGRATION_PLATFORMS[@]}" \
            --version-set "${VERSION_SETS_TO_TEST[@]}" \
            --partition-module pkg 1 \
            --partition-module sdk 1 \
            --partition-module tests 2 \
            --partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8
          )

          ACCEPTANCE_TEST_MATRIX_WIN="{}"
          if [[ " ${ACCEPTANCE_PLATFORMS[*]} " =~ [[:space:]]windows-latest[[:space:]] ]]; then
            ACCEPTANCE_TEST_MATRIX_WIN=$(
              ./scripts/get-job-matrix.py \
              -vvv \
              generate-matrix \
              --kind acceptance-test \
              "$CODEGEN_TESTS_FLAG" \
              --platform windows-latest \
              --version-set current \
              --partition-module pkg 1 \
              --partition-module sdk 1 \
              --partition-module tests 2 \
              --partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8
            )
          fi

          ACCEPTANCE_TEST_MATRIX_MACOS="{}"
          if [[ " ${ACCEPTANCE_PLATFORMS[*]} " =~ [[:space:]]macos-latest[[:space:]] ]]; then
            ACCEPTANCE_TEST_MATRIX_MACOS=$(
              ./scripts/get-job-matrix.py \
              -vvv \
              generate-matrix \
              --kind acceptance-test \
              "$CODEGEN_TESTS_FLAG" \
              --platform macos-latest \
              --version-set current \
              --partition-module pkg 1 \
              --partition-module sdk 1 \
              --partition-module tests 1 \
              --partition-package github.com/pulumi/pulumi/tests/integration tests/integration 2
            )
          fi
          echo "::endgroup::"

          echo "::group::Version set variable"
          VERSION_SET=$(./scripts/get-job-matrix.py \
            generate-version-set \
            --version-set current
          )
          echo "::endgroup::"

          echo "::group::Unit test matrix"
          echo "$UNIT_TEST_MATRIX" | yq -P '.'
          echo "::endgroup::"
          echo "::group::Integration test matrix"
          echo "$INTEGRATION_TEST_MATRIX" | yq -P '.'
          echo "::endgroup::"
          echo "::group::acceptance test matrix windows"
          echo "$ACCEPTANCE_TEST_MATRIX_WIN" | yq -P '.'
          echo "::endgroup::"
          echo "::group::acceptance test matrix macos"
          echo "$ACCEPTANCE_TEST_MATRIX_MACOS" | yq -P '.'
          echo "::endgroup::"
          echo "::group::Version set"
          echo "$VERSION_SET" | yq -P '.'
          echo "::endgroup::"

          echo "::group::Set outputs"
          ./.github/scripts/set-output unit-test-matrix "${UNIT_TEST_MATRIX}"
          ./.github/scripts/set-output integration-test-matrix "${INTEGRATION_TEST_MATRIX}"
          ./.github/scripts/set-output acceptance-test-matrix-win "${ACCEPTANCE_TEST_MATRIX_WIN}"
          ./.github/scripts/set-output acceptance-test-matrix-macos "${ACCEPTANCE_TEST_MATRIX_MACOS}"
          ./.github/scripts/set-output version-set "${VERSION_SET}"
          ./.github/scripts/set-output build-targets "${BUILD_TARGETS}"
          echo "::endgroup::"          
    outputs:
      unit-test-matrix: "${{ fromJson(steps.matrix.outputs.unit-test-matrix) }}"
      integration-test-matrix: "${{ fromJson(steps.matrix.outputs.integration-test-matrix) }}"
      acceptance-test-matrix-win: "${{ fromJson(steps.matrix.outputs.acceptance-test-matrix-win) }}"
      acceptance-test-matrix-macos: "${{ fromJson(steps.matrix.outputs.acceptance-test-matrix-macos) }}"
      version-set: "${{ fromJson(steps.matrix.outputs.version-set) }}"
      build-targets: "${{ fromJson(steps.matrix.outputs.build-targets) }}"

  lint:
    name: Lint
    needs: [matrix]
    if: ${{ inputs.lint }}
    uses: ./.github/workflows/ci-lint.yml
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
    with:
      ref: ${{ inputs.ref }}
      version-set: ${{ needs.matrix.outputs.version-set }}

  build-binaries:
    name: build binaries
    needs: [matrix]
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
      matrix:
        target: ${{ fromJson(needs.matrix.outputs.build-targets) }}
    uses: ./.github/workflows/ci-build-binaries.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      os: ${{ matrix.target.os }}
      arch: ${{ matrix.target.arch }}
      build-platform: ${{ matrix.target.build-platform }}
      version-set: ${{ needs.matrix.outputs.version-set }}
      enable-coverage: ${{ inputs.enable-coverage }}
      # Windows and Linux need CGO and cross compile support to support -race. So for now only enable it for darwin and amd64 linux.
      enable-race-detection: ${{ matrix.target.os == 'darwin' || (matrix.target.os == 'linux' && matrix.target.arch == 'amd64') }}
      artifact-suffix: '-integration'
    secrets: inherit

  build-display-wasm-module:
    name: build display WASM module
    needs: [matrix]
    uses: ./.github/workflows/ci-build-binaries.yml
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      os: js
      arch: wasm
      build-platform: "ubuntu-latest"
      version-set: ${{ needs.matrix.outputs.version-set }}
    secrets: inherit

  build-sdks:
    name: Build SDKs
    needs: [matrix]
    uses: ./.github/workflows/ci-build-sdks.yml
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      version-set: ${{ needs.matrix.outputs.version-set }}
    secrets: inherit

  # Tests that can run concurrently with builds.
  unit-test:
    # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
    # appearing in the rendered title of the job name: It changes this:
    #   CI / Unit Test (cd sdk/dotnet && make dotnet_test, cd sdk/dotnet && make dotnet_test, macos-11, mi... / sdk/dotnet dotnet_test on macos-11/current
    #   (See: https://github.com/pulumi/pulumi/runs/8241055084?check_suite_focus=true#logs)
    # To this:
    #   CI / Unit Test / sdk/dotnet dotnet_test on macos-11/current
    name: Unit Test${{ matrix.platform && '' }}
    needs: [matrix]
    if: ${{ needs.matrix.outputs.unit-test-matrix != '{}' }}
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
      matrix: ${{ fromJson(needs.matrix.outputs.unit-test-matrix) }}
    uses: ./.github/workflows/ci-run-test.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      platform: ${{ matrix.platform }}

      test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
      test-command: ${{ matrix.test-suite.command }}
      is-integration-test: false
      enable-coverage: ${{ inputs.enable-coverage }}
      test-retries: ${{ inputs.test-retries }}
      # require-build: false # TODO, remove ${{ matrix.require-build || false }}

      version-set: ${{ toJson(matrix.version-set) }}
    secrets: inherit

  # Tests that depend on builds
  integration-test:
    # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
    # appearing in the rendered title of the job name. See: unit test.
    name: Integration Test${{ matrix.platform && '' }}
    needs: [matrix, build-binaries, build-sdks]
    if: ${{ needs.matrix.outputs.integration-test-matrix != '{}' }}
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
      matrix: ${{ fromJson(needs.matrix.outputs.integration-test-matrix) }}
    uses: ./.github/workflows/ci-run-test.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      platform: ${{ matrix.platform }}

      test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
      test-command: ${{ matrix.test-suite.command }}
      is-integration-test: true
      enable-coverage: ${{ inputs.enable-coverage }}
      test-retries: ${{ inputs.test-retries }}
      # require-build: false # TODO, remove ${{ matrix.require-build || false }}

      version-set: ${{ toJson(matrix.version-set) }}
    secrets: inherit

  # Tests that depend on builds, but a smaller subset against Windows platform.
  acceptance-test-win:
    # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
    # appearing in the rendered title of the job name. See: unit test.
    name: Acceptance Test${{ matrix.platform && '' }}
    needs: [matrix, build-binaries, build-sdks]
    if: ${{ needs.matrix.outputs.acceptance-test-matrix-win != '{}' }}
    # allow jobs to fail if the platform contains windows
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
      matrix: ${{ fromJson(needs.matrix.outputs.acceptance-test-matrix-win) }}
    uses: ./.github/workflows/ci-run-test.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      platform: ${{ matrix.platform }}

      test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
      test-command: ${{ matrix.test-suite.command }}
      is-integration-test: true
      enable-coverage: false
      test-retries: ${{ inputs.test-retries }}
      # require-build: false # TODO, remove ${{ matrix.require-build || false }}

      version-set: ${{ toJson(matrix.version-set) }}
    secrets: inherit

  # Tests that depend on builds, but a smaller subset against MacOS platform.
  acceptance-test-macos:
    # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
    # appearing in the rendered title of the job name. See: unit test.
    name: Acceptance Test${{ matrix.platform && '' }}
    needs: [matrix, build-binaries, build-sdks]
    if: ${{ needs.matrix.outputs.acceptance-test-matrix-macos != '{}' }}
    # allow jobs to fail if the platform contains windows
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
      matrix: ${{ fromJson(needs.matrix.outputs.acceptance-test-matrix-macos) }}
    uses: ./.github/workflows/ci-run-test.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      platform: ${{ matrix.platform }}

      test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
      test-command: ${{ matrix.test-suite.command }}
      is-integration-test: true
      enable-coverage: false
      test-retries: ${{ inputs.test-retries }}
      # require-build: false # TODO, remove ${{ matrix.require-build || false }}

      version-set: ${{ toJson(matrix.version-set) }}
    secrets: inherit

  test-collect-reports:
    needs: [unit-test, integration-test, acceptance-test-macos]
    if: ${{ always() }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/cache@v4
        with:
          path: test-results
          key: gotestsum-timing-${{ github.run_number }}
          restore-keys: gotestsum-timing-
      - uses: actions/download-artifact@v4
        continue-on-error: true
        with:
          pattern: gotestsum-test-results-*
          merge-multiple: true
          path: test-results
      - name: List and clean up test results
        continue-on-error: true
        run: |
          ls -lhR test-results
          find test-results -mindepth 1 -name '*.json' -mtime +7 -delete          
  test-collect-coverage:
    needs: [unit-test, integration-test]
    if: ${{ inputs.enable-coverage }}
    runs-on: ubuntu-latest
    steps:
      # Check that there are no failed tests.
      - name: Check tests completed successfully.
        run: |
          JSON='${{ toJson(needs) }}'
          if test -z "$(echo $JSON | jq -r '.[] | .result' | grep -v 'success')"; then
              echo "Tests OK!"
          else
              echo "Test Failure.. skipping CodeCov upload."
              exit 1
          fi          
      # Checkout repository to upload coverage results.
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
      - name: Retrieve code coverage reports
        uses: actions/download-artifact@v4
        with:
          pattern: coverage-*
          merge-multiple: true
          path: coverage
      # For PRs we need to set the merge base manually, otherwise it is sometimes incorrect.  See https://github.com/pulumi/pulumi/pull/17204#issuecomment-2405599310
      - name: Set codecov PR base
        if: github.event_name == 'pull_request'
        run: |
          curl -Os https://cli.codecov.io/latest/linux/codecov
          sudo chmod +x codecov
          ./codecov pr-base-picking --base-sha "$(git merge-base origin/master HEAD)" --pr ${{ github.event.number }} --slug pulumi/pulumi --token ${{ secrets.CODECOV_TOKEN }} --service github          
      - name: Upload code coverage
        uses: codecov/codecov-action@v4
        with:
          directory: coverage/
          files: "*,!.gitkeep"
          fail_ci_if_error: false
          verbose: true
          token: ${{ secrets.CODECOV_TOKEN }}
  build-release-binaries:
    # This overwrites the previously built testing binaries with versions without coverage.
    name: Rebuild binaries
    needs: [matrix]
    if: ${{ inputs.enable-coverage }}
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
      matrix:
        target: ${{ fromJson(needs.matrix.outputs.build-targets) }}
    uses: ./.github/workflows/ci-build-binaries.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      os: ${{ matrix.target.os }}
      arch: ${{ matrix.target.arch }}
      build-platform: ${{ matrix.target.build-platform }}
      version-set: ${{ needs.matrix.outputs.version-set }}
      enable-coverage: false
    secrets: inherit