pulumi/.github/workflows/ci-run-test.yml

name: Test

permissions:
  contents: read

on:
  workflow_call:
    inputs:
      ref:
        required: true
        description: "GitHub ref to use"
        type: string
      version:
        required: true
        description: "Version to produce"
        type: string
      platform:
        description: "OS to run tests on, e.g.: ubuntu-latest"
        required: true
        type: string

      test-name:
        description: "Name of the test to run"
        required: true
        type: string
      test-command:
        description: Test command to run
        required: true
        type: string
      is-integration-test:
        description: Whether to download and install build artifacts
        required: false
        default: false
        type: boolean
      is-performance-test:
        description: Whether to download and install build artifacts for performance tests
        required: false
        default: false
        type: boolean
      enable-coverage:
        description: "Collects coverage stats; requires cov-enabled builds"
        default: false
        required: false
        type: boolean
      test-retries:
        description: "How often tests are retried"
        default: 0
        required: false
        type: number

      version-set:
        required: false
        description: "Set of language versions to use for builds, lints, releases, etc."
        type: string
        # Example provided for illustration, this value is derived by scripts/get-job-matrix.py build
        default: |
          {
            "dotnet": "6.0.x",
            "go": "1.18.x",
            "nodejs": "16.x",
            "python": "3.9.x"
          }          
      continue-on-error:
        description: "Whether to continue running the job if the step fails"
        required: false
        default: false
        type: boolean

defaults:
  run:
    shell: bash

env:
  PULUMI_VERSION: ${{ inputs.version }}
  PULUMI_TEST_OWNER: "moolumi"
  PULUMI_TEST_ORG: "moolumi"
  PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_PROD_ACCESS_TOKEN }}
  # Release builds use the service, PR checks and snapshots will use the local backend if possible.
  PULUMI_TEST_USE_SERVICE: ${{ !contains(inputs.version, '-') }}
  # We're hitting a lot of github limits because of deploytest trying to auto install plugins, skip that for now.
  PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION: "true"
  PYTHON: python
  GO_TEST_PARALLELISM: 8
  GO_TEST_PKG_PARALLELISM: 2
  GO_TEST_SHUFFLE: off
  PULUMI_TEST_RETRIES: ${{ inputs.test-retries }}
  DOTNET_CLI_TELEMETRY_OPTOUT: "true"
  DOTNET_ROLL_FORWARD: "Major"
  SEGMENT_DOWNLOAD_TIMEOUT_MIN: 10

jobs:
  test:
    name: ${{ inputs.test-name }}
    env:
      PULUMI_HOME: ${{ github.workspace }}/home
      TEST_ALL_DEPS: ""

    runs-on: ${{ inputs.platform }}

    timeout-minutes: 60
    continue-on-error: ${{ inputs.continue-on-error }}
    steps:
      - if: contains(inputs.platform, 'ubuntu')
        name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          swap-storage: false
          large-packages: false
          dotnet: false

      # Create a coverage file to register a testing job was started.
      # This artifact will be overwritten with "OK" at the end of the job.
      - name: Write test started file
        shell: bash
        run: |
          HASH=$(echo '${{ inputs.test-name }}-${{ inputs.is-integration-test && 'integration' || 'unit' }}' | git hash-object --literally --stdin | awk '{ print $1 }')
          echo "TEST_HASH_ID=${HASH}" >> "$GITHUB_ENV"          
      - name: "Windows cache workaround"
        # https://github.com/actions/cache/issues/752#issuecomment-1222415717
        # but only modify the path by adding tar.exe
        if: ${{ runner.os == 'Windows' }}
        env:
          TOOL_BIN: ${{ runner.temp }}/tar-bin
        run: |
          set -x
          mkdir -p "${TOOL_BIN}"
          cp "C:/Program Files/Git/usr/bin/tar.exe" "${TOOL_BIN}"
          PATH="${TOOL_BIN}:${PATH}"
          echo "$TOOL_BIN" | tee -a "$GITHUB_PATH"
          command -v tar
          tar --version          
      - name: Reduce Windows test parallelism
        if: ${{ runner.os == 'Windows' }}
        run: |
          {
            echo "GO_TEST_PARALLELISM=4"
            echo "GO_TEST_PKG_PARALLELISM=1"
            echo "GO_TEST_RACE=false"
          } >> "$GITHUB_ENV"
          # For debugging:
          ps aux          
      - name: "macOS use coreutils"
        if: ${{ runner.os == 'macOS' }}
        run: |
          set -x
          brew install coreutils
          echo "/usr/local/opt/coreutils/libexec/gnubin" | tee -a "$GITHUB_PATH"
          command -v bash
          bash --version          
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
      - name: Setup versioning env vars
        env:
          version: ${{ inputs.version }}
        run: |
          ./scripts/versions.sh | tee -a "$GITHUB_ENV"          
      - name: Enable code coverage
        if: ${{ inputs.enable-coverage }}
        run: |
          echo "PULUMI_TEST_COVERAGE_PATH=$(pwd)/coverage" >> "$GITHUB_ENV"
          # Post integration test coverage to a temporary directory.
          # This will be merged at a later step.
          echo "GOCOVERDIR=$(mktemp -d)" >> "$GITHUB_ENV"          
      - name: Configure Go Cache Key
        env:
          CACHE_KEY: "${{ fromJson(inputs.version-set).go }}-${{ runner.os }}-${{ runner.arch }}"
        run: echo "$CACHE_KEY" > .gocache.tmp
      - uses: actions/setup-go@v5
        id: setup-go
        env:
          SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2
        with:
          go-version: ${{ fromJson(inputs.version-set).go }}
          check-latest: true
          cache: true
          cache-dependency-path: |
            pkg/go.sum
            .gocache.tmp            
      - name: Prime Go cache
        if: ${{ ! steps.setup-go.outputs.cache-hit }}
        # Compile every test to ensure we populate a good cache entry.
        run: |
          ( cd sdk && go test -run "_________" ./... )
          ( cd pkg && go test -run "_________" ./... )          
      - name: Install delve
        run: |
          go install github.com/go-delve/delve/cmd/dlv@latest          
      - name: Set up Python ${{ fromJson(inputs.version-set).python }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ fromJson(inputs.version-set).python }}
          cache: pip
          cache-dependency-path: sdk/python/requirements.txt
      - name: Set up DotNet ${{ fromJson(inputs.version-set).dotnet }}
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: ${{ fromJson(inputs.version-set).dotnet }}
          dotnet-quality: ga
      - name: Set up Node ${{ fromJson(inputs.version-set).nodejs }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ fromJson(inputs.version-set).nodejs }}
          cache: yarn
          cache-dependency-path: sdk/nodejs/yarn.lock
      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          java-version: '11'
          distribution: 'temurin'
      - name: Setup Gradle
        uses: gradle/actions/setup-gradle@v3
        with:
          gradle-version: "7.6"
      - name: Uninstall pre-installed Pulumi (windows)
        if: inputs.platform == 'windows-latest'
        run: |
          if command -v pulumi.exe; then
            echo "Deleting pulumi"
            rm -rf "$(command -v pulumi.exe)/../pulumi*"
          fi          
      - name: Install yarn and pnpm
        run: |
          npm install -g yarn pnpm          
      - name: Install Python deps
        run: |
          python -m pip install --upgrade pip requests wheel urllib3 chardet          
      - name: Install poetry
        run: pipx install poetry
      - name: Setup git
        run: |
          git config --global user.email "you@example.com"
          git config --global user.name "Your Name"          
      - name: Set Go Dep path
        run: |
          echo "PULUMI_GO_DEP_ROOT=$(dirname "$(pwd)")" | tee -a "${GITHUB_ENV}"          
      - name: Install gotestsum
        uses: jaxxstorm/action-install-gh-release@v1.11.0
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          repo: gotestyourself/gotestsum
          tag: v1.8.1
          cache: enable
      - name: Install goteststats
        uses: jaxxstorm/action-install-gh-release@v1.11.0
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          repo: pulumi/goteststats
          tag: v0.0.7
          cache: enable
      - name: Generate artifact name
        if: ${{ inputs.is-integration-test || inputs.is-performance-test }}
        id: goenv
        shell: bash
        run: |
          echo "CLI-TARGET=$(go env GOOS)-$(go env GOARCH)" >> "${GITHUB_OUTPUT}"          

      # Ensure tests do not rely on pre-installed packages in CI. Unit tests must run absent a local
      # Pulumi install, and integration tests must only test the version built by CI.
      - name: Remove Pre-installed Pulumi
        env:
          RUNNER_OS: ${{ runner.os }}
        run: |
          EXT=""
          if [ "$RUNNER_OS" == "Windows" ]; then
            EXT=".exe"
          fi

          if command -v "pulumi${EXT}"; then
            PULUMI_INSTALL_DIR=$(dirname "$(command -v "pulumi${EXT}")")
            echo "Deleting Pulumi"
            rm -v "$PULUMI_INSTALL_DIR"/pulumi*
          fi          

      # Integration and performance test only steps:
      - name: Download pulumi-${{ steps.goenv.outputs.cli-target }}
        if: ${{ inputs.is-integration-test }}
        uses: actions/download-artifact@v4
        with:
          name: artifacts-cli-${{ steps.goenv.outputs.cli-target }}-integration
          path: artifacts/cli
      - name: Download pulumi-${{ steps.goenv.outputs.cli-target }}
        if: ${{ inputs.is-performance-test }}
        uses: actions/download-artifact@v4
        with:
          name: artifacts-cli-${{ steps.goenv.outputs.cli-target }}-perf
          path: artifacts/cli
      - name: Install Pulumi Go Binaries
        if: ${{ inputs.is-integration-test || inputs.is-performance-test }}
        run: |
          echo "Checking contents of $PWD/artifacts/cli"
          find "$PWD/artifacts/cli"
          TMPDIR="$(mktemp -d)"

          mkdir -p bin

          # Extract files to temporary directory:
          find "$PWD/artifacts/cli" -name '*.zip' -print0 -exec unzip {} -d "$TMPDIR" \;
          find "$PWD/artifacts/cli" -name '*.tar.gz' -print0 -exec tar -xzvf {} -C "$TMPDIR" \;

          # Windows .zip files have an extra "bin" path part, support both:
          if [ -d "$TMPDIR/pulumi/bin" ]; then
            mv "${TMPDIR}/pulumi/bin/"* "$PWD/bin/"
          else
            mv "${TMPDIR}/pulumi/"* "$PWD/bin/"
          fi

          echo "Checking contents of $PWD/bin"
          find "$PWD/bin"

          LOCAL_PATH=$(./scripts/normpath "${{ github.workspace }}/bin")
          echo "Adding LOCAL_PATH=$LOCAL_PATH to PATH"
          echo "$LOCAL_PATH" >> "$GITHUB_PATH"          

        # /end integration and performance test steps
      - name: Verify Pulumi Version
        run: |
          command -v pulumi || echo "no pulumi"
          pulumi version    || echo "no pulumi"          
      - name: Ensure dependencies for the Node SDK
        run: |
          cd sdk/nodejs
          make ensure          
      - name: Build the Node SDK
        run: |
          cd sdk/nodejs
          make build_package
          cd bin
          yarn link          
      - name: Ensure dependencies for the Python SDK
        run: |
          cd sdk/python
          make ensure          
      - name: Install Python SDK
        run: |
          cd sdk/python
          make build_package          
      - name: Set PULUMI_ACCEPT if version-set is not current
        if: ${{ fromJson(inputs.version-set).name != 'current' }}
        run: echo "PULUMI_ACCEPT=TRUE" >> "${GITHUB_ENV}"
      - name: Create GCP service account key file
        run: |
          echo '${{ secrets.GCP_SERVICE_ACCOUNT }}' > '${{ runner.temp }}/application_default_credentials.json'          
      - name: run tests "${{ inputs.test-command }}"
        id: test
        run: ${{ inputs.test-command }}
        env:
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          AZURE_STORAGE_SAS_TOKEN: ${{ secrets.AZURE_STORAGE_SAS_TOKEN }}
          PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules
          PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GOOGLE_APPLICATION_CREDENTIALS: ${{ runner.temp }}/application_default_credentials.json
          AWS_ACCESS_KEY: ${{ secrets.AWS_CI_ACCESS_KEY }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_ACCESS_SECRET }}
      - name: Convert Node coverage data
        if: ${{ inputs.platform != 'windows-latest' }}
        run: |
          cd sdk/nodejs
          if [ -e .nyc_output ]; then
            UUID=$(python -c "import uuid; print(str(uuid.uuid4()).replace('-', '').lower())")
            COVERAGE_PATH="${PULUMI_TEST_COVERAGE_PATH:-./coverage}"
            yarn run nyc report -r cobertura --report-dir "$COVERAGE_PATH"
            mv "$COVERAGE_PATH/cobertura-coverage.xml" "$COVERAGE_PATH/cobertura-coverage-$UUID.xml"
          fi          
      - name: Merge integration test code coverage
        if: ${{ inputs.enable-coverage }}
        run: |
          # Merge coverage data from coverage-instrumented binaries
          # if available.
          if [ -n "$(ls -A "$GOCOVERDIR")" ]; then
            # Cross-platform way to get milliseconds since Unix epoch.
            UUID=$(python -c "import uuid; print(str(uuid.uuid4()).replace('-', '').lower())")
            go tool covdata textfmt -i="$GOCOVERDIR" -o="$PULUMI_TEST_COVERAGE_PATH/integration-$UUID.cov"
          fi          
      - name: Upload code coverage
        uses: actions/upload-artifact@v4
        if: ${{ inputs.enable-coverage }}
        with:
          name: coverage-${{ env.TEST_HASH_ID }}
          path: |
            coverage/*
            !coverage/.gitkeep            
      - name: Summarize Test Time by Package
        continue-on-error: true
        env:
          RUNNER_OS: ${{ runner.os }}
        run: |
          mkdir -p test-results
          touch test-results/empty.json # otherwise goteststats fails below when no files match
          # Remove output lines, they make analysis slower & could leak logs:
          if [ "$RUNNER_OS" == "macOS" ]; then
            # It's just another case of BSD sed, GNU sed.
            sed -i '' -e '/"Action":"output"/d' ./test-results/*.json
          else
            sed -i'' -e '/"Action":"output"/d' ./test-results/*.json
          fi

          goteststats -statistic pkg-time test-results/*.json          
      - name: Summarize Test Times by Individual Test
        continue-on-error: true
        run: |
          goteststats -statistic test-time test-results/*.json | head -n 100 || \
                      if [[ $? -eq 141 ]]; then true; else exit $?; fi          
      - name: Upload artifacts
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: gotestsum-test-results-${{ env.TEST_HASH_ID }}
          path: test-results/*.json