pulumi/.github/workflows/ci-performance-gate.yml

# This workflow runs performance tests to ensure we do not introduce performance
# regressions. This runs for every PR, as well as on every merge to the master
# branch.
#
# The Pulumi CLI binaries used in this workflow are built without race detection
# or coverage instrumentation to ensure that the performance tests are not
# affected by these flags.

name: Performance Gate

permissions:
  contents: read

on:
  workflow_call:
    inputs:
      ref:
        required: true
        description: "GitHub ref to use"
        type: string
      version:
        required: true
        description: "Version to produce"
        type: string
      test-version-sets:
        required: false
        default: minimum current
        description: Version sets on which to run integration tests
        type: string
      performance-test-platforms:
        required: false
        default: ubuntu-latest
        description: Platforms on which to run performance tests, as a space delimited list
        type: string
      fail-fast:
        required: false
        default: false
        description: "Fail all workflows whenever one of them fails"
        type: boolean
      test-retries:
        required: false
        default: 0
        description: "Retry tests n times if there are failures"
        type: number
    secrets:
      PULUMI_PROD_ACCESS_TOKEN:
        required: false
        description: "Pulumi access token, required to run tests against the service"
      AZURE_TENANT_ID:
        required: false
        description: "Azure tenant ID, required to run tests against Azure"
      AZURE_CLIENT_ID:
        required: false
        description: "Azure client ID, required to run tests against Azure"
      AZURE_CLIENT_SECRET:
        required: false
        description: "Azure clients secret, needs to be rotated before 2025-12-21 (see the pulumi-test user in Azure portal)"
      AZURE_STORAGE_SAS_TOKEN:
        required: false
        description: "Azure storage SAS token, required to run tests against Azure"
      GCP_SERVICE_ACCOUNT:
        required: false
        description: "GCP service account, required to run tests against GCP"

jobs:
  matrix:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
      - name: build matrix
        id: matrix
        env:
          TEST_VERSION_SETS: ${{ inputs.test-version-sets }}
          INPUT_PERFORMANCE_TEST_PLATFORMS: ${{ inputs.performance-test-platforms }}
        run: |
          echo "::group::Test matrix variables"
          readarray -td' ' VERSION_SETS_TO_TEST < <(echo -n "$TEST_VERSION_SETS"); declare -p VERSION_SETS_TO_TEST;
          readarray -td' ' PERFORMANCE_TEST_PLATFORMS < <(echo -n "$INPUT_PERFORMANCE_TEST_PLATFORMS"); declare -p PERFORMANCE_TEST_PLATFORMS;
          BUILD_TARGETS='[
              { "os": "linux",   "arch": "amd64", "build-platform": "ubuntu-latest" },
              { "os": "windows", "arch": "amd64", "build-platform": "ubuntu-latest" },
              { "os": "darwin",  "arch": "arm64", "build-platform": "ubuntu-latest" }
          ]'

          PERFORMANCE_TEST_MATRIX=$(
            ./scripts/get-job-matrix.py \
            -vvv \
            generate-matrix \
            --kind performance-test \
            --platform "${PERFORMANCE_TEST_PLATFORMS[@]}" \
            --version-set "${VERSION_SETS_TO_TEST[@]}"
          )
          echo "::endgroup::"

          echo "::group::Version set variable"
          VERSION_SET=$(./scripts/get-job-matrix.py \
            generate-version-set \
            --version-set current
          )
          echo "::endgroup::"

          echo "::group::Performance test matrix"
          echo "$PERFORMANCE_TEST_MATRIX" | yq -P '.'
          echo "::endgroup::"
          echo "::group::Version set"
          echo "$VERSION_SET" | yq -P '.'
          echo "::endgroup::"

          echo "::group::Set outputs"
          ./.github/scripts/set-output performance-test-matrix "${PERFORMANCE_TEST_MATRIX}"
          ./.github/scripts/set-output version-set "${VERSION_SET}"
          ./.github/scripts/set-output build-targets "${BUILD_TARGETS}"
          echo "::endgroup::"          
    outputs:
      performance-test-matrix: "${{ fromJson(steps.matrix.outputs.performance-test-matrix) }}"
      version-set: "${{ fromJson(steps.matrix.outputs.version-set) }}"
      build-targets: "${{ fromJson(steps.matrix.outputs.build-targets) }}"

  build-binaries:
    name: build binaries
    needs: [matrix]
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
      matrix:
        target: ${{ fromJson(needs.matrix.outputs.build-targets) }}
    uses: ./.github/workflows/ci-build-binaries.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      os: ${{ matrix.target.os }}
      arch: ${{ matrix.target.arch }}
      build-platform: ${{ matrix.target.build-platform }}
      version-set: ${{ needs.matrix.outputs.version-set }}
      # For performance tests, we do not want coverage or race detection enabled.
      enable-coverage: false
      enable-race-detection: false
      # Suffix the artifacts so they do not clash with those of the main build.
      artifact-suffix: '-perf'
    secrets: inherit

  performance-test:
    # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
    # appearing in the rendered title of the job name.
    name: Performance Test${{ matrix.platform && '' }}
    needs: [matrix, build-binaries]
    strategy:
      fail-fast: ${{ inputs.fail-fast }}
      matrix: ${{ fromJson(needs.matrix.outputs.performance-test-matrix) }}
    uses: ./.github/workflows/ci-run-test.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      platform: ${{ matrix.platform }}
      test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
      test-command: ${{ matrix.test-suite.command }}
      is-performance-test: true
      enable-coverage: false
      test-retries: ${{ inputs.test-retries }}
      version-set: ${{ toJson(matrix.version-set) }}
    secrets: inherit