name: Test permissions: contents: read on: workflow_call: inputs: ref: required: true description: "GitHub ref to use" type: string version: required: true description: "Version to produce" type: string platform: description: "OS to run tests on, e.g.: ubuntu-latest" required: true type: string test-name: description: "Name of the test to run" required: true type: string test-command: description: Test command to run required: true type: string is-integration-test: description: Whether to download and install build artifacts required: false default: false type: boolean enable-coverage: description: "Collects coverage stats; requires cov-enabled builds" default: false required: false type: boolean test-retries: description: "How often tests are retried" default: 0 required: false type: number version-set: required: false description: "Set of language versions to use for builds, lints, releases, etc." type: string # Example provided for illustration, this value is derived by scripts/get-job-matrix.py build default: | { "dotnet": "6.0.x", "go": "1.18.x", "nodejs": "16.x", "python": "3.9.x" } continue-on-error: description: "Whether to continue running the job if the step fails" required: false default: false type: boolean defaults: run: shell: bash env: PULUMI_VERSION: ${{ inputs.version }} PULUMI_TEST_OWNER: "moolumi" PULUMI_TEST_ORG: "moolumi" PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_PROD_ACCESS_TOKEN }} # Release builds use the service, PR checks and snapshots will use the local backend if possible. PULUMI_TEST_USE_SERVICE: ${{ !contains(inputs.version, '-') }} # We're hitting a lot of github limits because of deploytest trying to auto install plugins, skip that for now. PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION: "true" PYTHON: python GO_TEST_PARALLELISM: 8 GO_TEST_PKG_PARALLELISM: 2 GO_TEST_SHUFFLE: off PULUMI_TEST_RETRIES: ${{ inputs.test-retries }} DOTNET_CLI_TELEMETRY_OPTOUT: "true" DOTNET_ROLL_FORWARD: "Major" SEGMENT_DOWNLOAD_TIMEOUT_MIN: 10 jobs: test: name: ${{ inputs.test-name }} env: PULUMI_HOME: ${{ github.workspace }}/home TEST_ALL_DEPS: "" runs-on: ${{ inputs.platform }} timeout-minutes: 60 continue-on-error: ${{ inputs.continue-on-error }} steps: # Create a coverage file to register a testing job was started. # This artifact will be overwritten with "OK" at the end of the job. - name: Write test started file shell: bash run: | HASH=$(echo '${{ inputs.test-name }}-${{ inputs.is-integration-test && 'integration' || 'unit' }}' | git hash-object --literally --stdin | awk '{ print $1 }') echo "TEST_HASH_ID=${HASH}" >> "$GITHUB_ENV" - name: "Windows cache workaround" # https://github.com/actions/cache/issues/752#issuecomment-1222415717 # but only modify the path by adding tar.exe if: ${{ runner.os == 'Windows' }} env: TOOL_BIN: ${{ runner.temp }}/tar-bin run: | set -x mkdir -p "${TOOL_BIN}" cp "C:/Program Files/Git/usr/bin/tar.exe" "${TOOL_BIN}" PATH="${TOOL_BIN}:${PATH}" echo "$TOOL_BIN" | tee -a "$GITHUB_PATH" command -v tar tar --version - name: Reduce Windows test parallelism if: ${{ runner.os == 'Windows' }} run: | { echo "GO_TEST_PARALLELISM=4" echo "GO_TEST_PKG_PARALLELISM=1" echo "GO_TEST_RACE=false" } >> "$GITHUB_ENV" # For debugging: ps aux - name: "macOS use coreutils" if: ${{ runner.os == 'macOS' }} run: | set -x brew install coreutils echo "/usr/local/opt/coreutils/libexec/gnubin" | tee -a "$GITHUB_PATH" command -v bash bash --version - uses: actions/checkout@v4 with: ref: ${{ inputs.ref }} - name: Setup versioning env vars env: version: ${{ inputs.version }} run: | ./scripts/versions.sh | tee -a "$GITHUB_ENV" - name: Enable code coverage if: ${{ inputs.enable-coverage }} run: | echo "PULUMI_TEST_COVERAGE_PATH=$(pwd)/coverage" >> "$GITHUB_ENV" # Post integration test coverage to a temporary directory. # This will be merged at a later step. echo "GOCOVERDIR=$(mktemp -d)" >> "$GITHUB_ENV" - name: Configure Go Cache Key env: CACHE_KEY: "${{ fromJson(inputs.version-set).go }}-${{ runner.os }}-${{ runner.arch }}" run: echo "$CACHE_KEY" > .gocache.tmp - uses: actions/setup-go@v5 id: setup-go env: SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2 with: go-version: ${{ fromJson(inputs.version-set).go }} check-latest: true cache: true cache-dependency-path: | pkg/go.sum .gocache.tmp - name: Prime Go cache if: ${{ ! steps.setup-go.outputs.cache-hit }} # Compile every test to ensure we populate a good cache entry. run: | ( cd sdk && go test -run "_________" ./... ) ( cd pkg && go test -run "_________" ./... ) - name: Set up Python ${{ fromJson(inputs.version-set).python }} uses: actions/setup-python@v5 with: python-version: ${{ fromJson(inputs.version-set).python }} cache: pip cache-dependency-path: sdk/python/requirements.txt - name: Set up DotNet ${{ fromJson(inputs.version-set).dotnet }} uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ fromJson(inputs.version-set).dotnet }} dotnet-quality: ga - name: Set up Node ${{ fromJson(inputs.version-set).nodejs }} uses: actions/setup-node@v4 with: node-version: ${{ fromJson(inputs.version-set).nodejs }} cache: yarn cache-dependency-path: sdk/nodejs/yarn.lock - name: Set up JDK 11 uses: actions/setup-java@v4 with: java-version: '11' distribution: 'temurin' - name: Setup Gradle uses: gradle/actions/setup-gradle@v3 with: gradle-version: "7.6" - name: Uninstall pre-installed Pulumi (windows) if: inputs.platform == 'windows-latest' run: | if command -v pulumi.exe; then echo "Deleting pulumi" rm -rf "$(command -v pulumi.exe)/../pulumi*" fi - name: Install yarn and pnpm run: | npm install -g yarn pnpm - name: Install Python deps run: | python -m pip install --upgrade pip requests wheel urllib3 chardet - name: Install poetry run: pipx install poetry - name: Setup git run: | git config --global user.email "you@example.com" git config --global user.name "Your Name" - name: Set Go Dep path run: | echo "PULUMI_GO_DEP_ROOT=$(dirname "$(pwd)")" | tee -a "${GITHUB_ENV}" - name: Install gotestsum uses: jaxxstorm/action-install-gh-release@v1.11.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: repo: gotestyourself/gotestsum tag: v1.8.1 cache: enable - name: Install goteststats uses: jaxxstorm/action-install-gh-release@v1.11.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: repo: pulumi/goteststats tag: v0.0.7 cache: enable - name: Generate artifact name id: goenv shell: bash run: | echo "CLI-TARGET=$(go env GOOS)-$(go env GOARCH)" >> "${GITHUB_OUTPUT}" # Ensure tests do not rely on pre-installed packages in CI. Unit tests must run absent a local # Pulumi install, and integration tests must only test the version built by CI. - name: Remove Pre-installed Pulumi env: RUNNER_OS: ${{ runner.os }} run: | EXT="" if [ "$RUNNER_OS" == "Windows" ]; then EXT=".exe" fi if command -v "pulumi${EXT}"; then PULUMI_INSTALL_DIR=$(dirname "$(command -v "pulumi${EXT}")") echo "Deleting Pulumi" rm -v "$PULUMI_INSTALL_DIR"/pulumi* fi # Integration test only steps: - name: Download pulumi-${{ steps.goenv.outputs.cli-target }} if: ${{ inputs.is-integration-test }} uses: actions/download-artifact@v4 with: name: artifacts-cli-${{ steps.goenv.outputs.cli-target }} path: artifacts/cli - name: Install Pulumi Go Binaries if: ${{ inputs.is-integration-test }} run: | echo "Checking contents of $PWD/artifacts/cli" find "$PWD/artifacts/cli" TMPDIR="$(mktemp -d)" mkdir -p bin # Extract files to temporary directory: find "$PWD/artifacts/cli" -name '*.zip' -print0 -exec unzip {} -d "$TMPDIR" \; find "$PWD/artifacts/cli" -name '*.tar.gz' -print0 -exec tar -xzvf {} -C "$TMPDIR" \; # Windows .zip files have an extra "bin" path part, support both: if [ -d "$TMPDIR/pulumi/bin" ]; then mv "${TMPDIR}/pulumi/bin/"* "$PWD/bin/" else mv "${TMPDIR}/pulumi/"* "$PWD/bin/" fi echo "Checking contents of $PWD/bin" find "$PWD/bin" LOCAL_PATH=$(./scripts/normpath "${{ github.workspace }}/bin") echo "Adding LOCAL_PATH=$LOCAL_PATH to PATH" echo "$LOCAL_PATH" >> "$GITHUB_PATH" # /end integration test steps - name: Verify Pulumi Version run: | command -v pulumi || echo "no pulumi" pulumi version || echo "no pulumi" - name: Ensure dependencies for the Node SDK run: | cd sdk/nodejs make ensure - name: Build the Node SDK run: | cd sdk/nodejs make build_package cd bin yarn link - name: Ensure dependencies for the Python SDK run: | cd sdk/python make ensure - name: Install Python SDK run: | cd sdk/python make build_package - name: Set PULUMI_ACCEPT if version-set is not current if: ${{ fromJson(inputs.version-set).name != 'current' }} run: echo "PULUMI_ACCEPT=TRUE" >> "${GITHUB_ENV}" - name: Create GCP service account key file run: | echo '${{ secrets.GCP_SERVICE_ACCOUNT }}' > '${{ runner.temp }}/application_default_credentials.json' - name: run tests "${{ inputs.test-command }}" id: test run: ${{ inputs.test-command }} env: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_STORAGE_SAS_TOKEN: ${{ secrets.AZURE_STORAGE_SAS_TOKEN }} PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GOOGLE_APPLICATION_CREDENTIALS: ${{ runner.temp }}/application_default_credentials.json AWS_ACCESS_KEY: ${{ secrets.AWS_CI_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_ACCESS_SECRET }} - name: Convert Node coverage data if: ${{ inputs.platform != 'windows-latest' }} run: | cd sdk/nodejs if [ -e .nyc_output ]; then yarn run nyc report -r cobertura --report-dir "$PULUMI_TEST_COVERAGE_PATH"; fi - name: Merge integration test code coverage if: ${{ inputs.enable-coverage }} run: | # Merge coverage data from coverage-instrumented binaries # if available. if [ -n "$(ls -A "$GOCOVERDIR")" ]; then # Cross-platform way to get milliseconds since Unix epoch. TS=$(python -c 'import time; print(int(time.time() * 1000))') go tool covdata textfmt -i="$GOCOVERDIR" -o="$PULUMI_TEST_COVERAGE_PATH/integration.$TS.cov" fi - name: Upload code coverage uses: actions/upload-artifact@v4 if: ${{ inputs.enable-coverage }} with: name: coverage-${{ env.TEST_HASH_ID }} path: | coverage/* !coverage/.gitkeep - name: Summarize Test Time by Package continue-on-error: true env: RUNNER_OS: ${{ runner.os }} run: | mkdir -p test-results touch test-results/empty.json # otherwise goteststats fails below when no files match # Remove output lines, they make analysis slower & could leak logs: if [ "$RUNNER_OS" == "macOS" ]; then # It's just another case of BSD sed, GNU sed. sed -i '' -e '/"Action":"output"/d' ./test-results/*.json else sed -i'' -e '/"Action":"output"/d' ./test-results/*.json fi goteststats -statistic pkg-time test-results/*.json - name: Summarize Test Times by Individual Test continue-on-error: true run: | goteststats -statistic test-time test-results/*.json | head -n 100 || \ if [[ $? -eq 141 ]]; then true; else exit $?; fi - name: Upload artifacts if: ${{ always() }} uses: actions/upload-artifact@v4 with: name: gotestsum-test-results-${{ env.TEST_HASH_ID }} path: test-results/*.json