mirror of https://github.com/pulumi/pulumi.git
404 lines
14 KiB
YAML
404 lines
14 KiB
YAML
name: Test
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
ref:
|
|
required: true
|
|
description: "GitHub ref to use"
|
|
type: string
|
|
version:
|
|
required: true
|
|
description: "Version to produce"
|
|
type: string
|
|
platform:
|
|
description: "OS to run tests on, e.g.: ubuntu-latest"
|
|
required: true
|
|
type: string
|
|
|
|
test-name:
|
|
description: "Name of the test to run"
|
|
required: true
|
|
type: string
|
|
test-command:
|
|
description: Test command to run
|
|
required: true
|
|
type: string
|
|
is-integration-test:
|
|
description: Whether to download and install build artifacts
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
enable-coverage:
|
|
description: "Collects coverage stats; requires cov-enabled builds"
|
|
default: false
|
|
required: false
|
|
type: boolean
|
|
test-retries:
|
|
description: "How often tests are retried"
|
|
default: 0
|
|
required: false
|
|
type: number
|
|
|
|
version-set:
|
|
required: false
|
|
description: "Set of language versions to use for builds, lints, releases, etc."
|
|
type: string
|
|
# Example provided for illustration, this value is derived by scripts/get-job-matrix.py build
|
|
default: |
|
|
{
|
|
"dotnet": "6.0.x",
|
|
"go": "1.18.x",
|
|
"nodejs": "16.x",
|
|
"python": "3.9.x"
|
|
}
|
|
continue-on-error:
|
|
description: "Whether to continue running the job if the step fails"
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
env:
|
|
PULUMI_VERSION: ${{ inputs.version }}
|
|
PULUMI_TEST_OWNER: "moolumi"
|
|
PULUMI_TEST_ORG: "moolumi"
|
|
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_PROD_ACCESS_TOKEN }}
|
|
# Release builds use the service, PR checks and snapshots will use the local backend if possible.
|
|
PULUMI_TEST_USE_SERVICE: ${{ !contains(inputs.version, '-') }}
|
|
# We're hitting a lot of github limits because of deploytest trying to auto install plugins, skip that for now.
|
|
PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION: "true"
|
|
PYTHON: python
|
|
GO_TEST_PARALLELISM: 8
|
|
GO_TEST_PKG_PARALLELISM: 2
|
|
GO_TEST_SHUFFLE: off
|
|
PULUMI_TEST_RETRIES: ${{ inputs.test-retries }}
|
|
DOTNET_CLI_TELEMETRY_OPTOUT: "true"
|
|
DOTNET_ROLL_FORWARD: "Major"
|
|
SEGMENT_DOWNLOAD_TIMEOUT_MIN: 10
|
|
|
|
jobs:
|
|
test:
|
|
name: ${{ inputs.test-name }}
|
|
env:
|
|
PULUMI_HOME: ${{ github.workspace }}/home
|
|
TEST_ALL_DEPS: ""
|
|
|
|
runs-on: ${{ inputs.platform }}
|
|
|
|
timeout-minutes: 60
|
|
continue-on-error: ${{ inputs.continue-on-error }}
|
|
steps:
|
|
- if: contains(inputs.platform, 'ubuntu')
|
|
name: Free Disk Space (Ubuntu)
|
|
uses: jlumbroso/free-disk-space@main
|
|
with:
|
|
tool-cache: false
|
|
swap-storage: false
|
|
large-packages: false
|
|
dotnet: false
|
|
|
|
# Create a coverage file to register a testing job was started.
|
|
# This artifact will be overwritten with "OK" at the end of the job.
|
|
- name: Write test started file
|
|
shell: bash
|
|
run: |
|
|
HASH=$(echo '${{ inputs.test-name }}-${{ inputs.is-integration-test && 'integration' || 'unit' }}' | git hash-object --literally --stdin | awk '{ print $1 }')
|
|
echo "TEST_HASH_ID=${HASH}" >> "$GITHUB_ENV"
|
|
- name: "Windows cache workaround"
|
|
# https://github.com/actions/cache/issues/752#issuecomment-1222415717
|
|
# but only modify the path by adding tar.exe
|
|
if: ${{ runner.os == 'Windows' }}
|
|
env:
|
|
TOOL_BIN: ${{ runner.temp }}/tar-bin
|
|
run: |
|
|
set -x
|
|
mkdir -p "${TOOL_BIN}"
|
|
cp "C:/Program Files/Git/usr/bin/tar.exe" "${TOOL_BIN}"
|
|
PATH="${TOOL_BIN}:${PATH}"
|
|
echo "$TOOL_BIN" | tee -a "$GITHUB_PATH"
|
|
command -v tar
|
|
tar --version
|
|
- name: Reduce Windows test parallelism
|
|
if: ${{ runner.os == 'Windows' }}
|
|
run: |
|
|
{
|
|
echo "GO_TEST_PARALLELISM=4"
|
|
echo "GO_TEST_PKG_PARALLELISM=1"
|
|
echo "GO_TEST_RACE=false"
|
|
} >> "$GITHUB_ENV"
|
|
# For debugging:
|
|
ps aux
|
|
- name: "macOS use coreutils"
|
|
if: ${{ runner.os == 'macOS' }}
|
|
run: |
|
|
set -x
|
|
brew install coreutils
|
|
echo "/usr/local/opt/coreutils/libexec/gnubin" | tee -a "$GITHUB_PATH"
|
|
command -v bash
|
|
bash --version
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.ref }}
|
|
- name: Setup versioning env vars
|
|
env:
|
|
version: ${{ inputs.version }}
|
|
run: |
|
|
./scripts/versions.sh | tee -a "$GITHUB_ENV"
|
|
- name: Enable code coverage
|
|
if: ${{ inputs.enable-coverage }}
|
|
run: |
|
|
echo "PULUMI_TEST_COVERAGE_PATH=$(pwd)/coverage" >> "$GITHUB_ENV"
|
|
# Post integration test coverage to a temporary directory.
|
|
# This will be merged at a later step.
|
|
echo "GOCOVERDIR=$(mktemp -d)" >> "$GITHUB_ENV"
|
|
- name: Configure Go Cache Key
|
|
env:
|
|
CACHE_KEY: "${{ fromJson(inputs.version-set).go }}-${{ runner.os }}-${{ runner.arch }}"
|
|
run: echo "$CACHE_KEY" > .gocache.tmp
|
|
- uses: actions/setup-go@v5
|
|
id: setup-go
|
|
env:
|
|
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2
|
|
with:
|
|
go-version: ${{ fromJson(inputs.version-set).go }}
|
|
check-latest: true
|
|
cache: true
|
|
cache-dependency-path: |
|
|
pkg/go.sum
|
|
.gocache.tmp
|
|
- name: Prime Go cache
|
|
if: ${{ ! steps.setup-go.outputs.cache-hit }}
|
|
# Compile every test to ensure we populate a good cache entry.
|
|
run: |
|
|
( cd sdk && go test -run "_________" ./... )
|
|
( cd pkg && go test -run "_________" ./... )
|
|
- name: Install delve
|
|
run: |
|
|
go install github.com/go-delve/delve/cmd/dlv@latest
|
|
- name: Set up Python ${{ fromJson(inputs.version-set).python }}
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ fromJson(inputs.version-set).python }}
|
|
cache: pip
|
|
cache-dependency-path: sdk/python/requirements.txt
|
|
- name: Set up DotNet ${{ fromJson(inputs.version-set).dotnet }}
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: ${{ fromJson(inputs.version-set).dotnet }}
|
|
dotnet-quality: ga
|
|
- name: Set up Node ${{ fromJson(inputs.version-set).nodejs }}
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: ${{ fromJson(inputs.version-set).nodejs }}
|
|
cache: yarn
|
|
cache-dependency-path: sdk/nodejs/yarn.lock
|
|
- name: Set up JDK 11
|
|
uses: actions/setup-java@v4
|
|
with:
|
|
java-version: '11'
|
|
distribution: 'temurin'
|
|
- name: Setup Gradle
|
|
uses: gradle/actions/setup-gradle@v3
|
|
with:
|
|
gradle-version: "7.6"
|
|
- name: Uninstall pre-installed Pulumi (windows)
|
|
if: inputs.platform == 'windows-latest'
|
|
run: |
|
|
if command -v pulumi.exe; then
|
|
echo "Deleting pulumi"
|
|
rm -rf "$(command -v pulumi.exe)/../pulumi*"
|
|
fi
|
|
- name: Install yarn and pnpm
|
|
run: |
|
|
npm install -g yarn pnpm
|
|
- name: Install Python deps
|
|
run: |
|
|
python -m pip install --upgrade pip requests wheel urllib3 chardet
|
|
- name: Install poetry
|
|
run: pipx install poetry
|
|
- name: Setup git
|
|
run: |
|
|
git config --global user.email "you@example.com"
|
|
git config --global user.name "Your Name"
|
|
- name: Set Go Dep path
|
|
run: |
|
|
echo "PULUMI_GO_DEP_ROOT=$(dirname "$(pwd)")" | tee -a "${GITHUB_ENV}"
|
|
- name: Install gotestsum
|
|
uses: jaxxstorm/action-install-gh-release@v1.11.0
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
with:
|
|
repo: gotestyourself/gotestsum
|
|
tag: v1.8.1
|
|
cache: enable
|
|
- name: Install goteststats
|
|
uses: jaxxstorm/action-install-gh-release@v1.11.0
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
with:
|
|
repo: pulumi/goteststats
|
|
tag: v0.0.7
|
|
cache: enable
|
|
- name: Generate artifact name
|
|
id: goenv
|
|
shell: bash
|
|
run: |
|
|
echo "CLI-TARGET=$(go env GOOS)-$(go env GOARCH)" >> "${GITHUB_OUTPUT}"
|
|
|
|
# Ensure tests do not rely on pre-installed packages in CI. Unit tests must run absent a local
|
|
# Pulumi install, and integration tests must only test the version built by CI.
|
|
- name: Remove Pre-installed Pulumi
|
|
env:
|
|
RUNNER_OS: ${{ runner.os }}
|
|
run: |
|
|
EXT=""
|
|
if [ "$RUNNER_OS" == "Windows" ]; then
|
|
EXT=".exe"
|
|
fi
|
|
|
|
if command -v "pulumi${EXT}"; then
|
|
PULUMI_INSTALL_DIR=$(dirname "$(command -v "pulumi${EXT}")")
|
|
echo "Deleting Pulumi"
|
|
rm -v "$PULUMI_INSTALL_DIR"/pulumi*
|
|
fi
|
|
|
|
# Integration test only steps:
|
|
- name: Download pulumi-${{ steps.goenv.outputs.cli-target }}
|
|
if: ${{ inputs.is-integration-test }}
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: artifacts-cli-${{ steps.goenv.outputs.cli-target }}
|
|
path: artifacts/cli
|
|
- name: Install Pulumi Go Binaries
|
|
if: ${{ inputs.is-integration-test }}
|
|
run: |
|
|
echo "Checking contents of $PWD/artifacts/cli"
|
|
find "$PWD/artifacts/cli"
|
|
TMPDIR="$(mktemp -d)"
|
|
|
|
mkdir -p bin
|
|
|
|
# Extract files to temporary directory:
|
|
find "$PWD/artifacts/cli" -name '*.zip' -print0 -exec unzip {} -d "$TMPDIR" \;
|
|
find "$PWD/artifacts/cli" -name '*.tar.gz' -print0 -exec tar -xzvf {} -C "$TMPDIR" \;
|
|
|
|
# Windows .zip files have an extra "bin" path part, support both:
|
|
if [ -d "$TMPDIR/pulumi/bin" ]; then
|
|
mv "${TMPDIR}/pulumi/bin/"* "$PWD/bin/"
|
|
else
|
|
mv "${TMPDIR}/pulumi/"* "$PWD/bin/"
|
|
fi
|
|
|
|
echo "Checking contents of $PWD/bin"
|
|
find "$PWD/bin"
|
|
|
|
LOCAL_PATH=$(./scripts/normpath "${{ github.workspace }}/bin")
|
|
echo "Adding LOCAL_PATH=$LOCAL_PATH to PATH"
|
|
echo "$LOCAL_PATH" >> "$GITHUB_PATH"
|
|
|
|
# /end integration test steps
|
|
- name: Verify Pulumi Version
|
|
run: |
|
|
command -v pulumi || echo "no pulumi"
|
|
pulumi version || echo "no pulumi"
|
|
- name: Ensure dependencies for the Node SDK
|
|
run: |
|
|
cd sdk/nodejs
|
|
make ensure
|
|
- name: Build the Node SDK
|
|
run: |
|
|
cd sdk/nodejs
|
|
make build_package
|
|
cd bin
|
|
yarn link
|
|
- name: Ensure dependencies for the Python SDK
|
|
run: |
|
|
cd sdk/python
|
|
make ensure
|
|
- name: Install Python SDK
|
|
run: |
|
|
cd sdk/python
|
|
make build_package
|
|
- name: Set PULUMI_ACCEPT if version-set is not current
|
|
if: ${{ fromJson(inputs.version-set).name != 'current' }}
|
|
run: echo "PULUMI_ACCEPT=TRUE" >> "${GITHUB_ENV}"
|
|
- name: Create GCP service account key file
|
|
run: |
|
|
echo '${{ secrets.GCP_SERVICE_ACCOUNT }}' > '${{ runner.temp }}/application_default_credentials.json'
|
|
- name: run tests "${{ inputs.test-command }}"
|
|
id: test
|
|
run: ${{ inputs.test-command }}
|
|
env:
|
|
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
|
|
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
|
|
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
|
|
AZURE_STORAGE_SAS_TOKEN: ${{ secrets.AZURE_STORAGE_SAS_TOKEN }}
|
|
PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules
|
|
PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
GOOGLE_APPLICATION_CREDENTIALS: ${{ runner.temp }}/application_default_credentials.json
|
|
AWS_ACCESS_KEY: ${{ secrets.AWS_CI_ACCESS_KEY }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_ACCESS_SECRET }}
|
|
- name: Convert Node coverage data
|
|
if: ${{ inputs.platform != 'windows-latest' }}
|
|
run: |
|
|
cd sdk/nodejs
|
|
if [ -e .nyc_output ]; then
|
|
UUID=$(python -c "import uuid; print(str(uuid.uuid4()).replace('-', '').lower())")
|
|
COVERAGE_PATH="${PULUMI_TEST_COVERAGE_PATH:-./coverage}"
|
|
yarn run nyc report -r cobertura --report-dir "$COVERAGE_PATH"
|
|
mv "$COVERAGE_PATH/cobertura-coverage.xml" "$COVERAGE_PATH/cobertura-coverage-$UUID.xml"
|
|
fi
|
|
- name: Merge integration test code coverage
|
|
if: ${{ inputs.enable-coverage }}
|
|
run: |
|
|
# Merge coverage data from coverage-instrumented binaries
|
|
# if available.
|
|
if [ -n "$(ls -A "$GOCOVERDIR")" ]; then
|
|
# Cross-platform way to get milliseconds since Unix epoch.
|
|
UUID=$(python -c "import uuid; print(str(uuid.uuid4()).replace('-', '').lower())")
|
|
go tool covdata textfmt -i="$GOCOVERDIR" -o="$PULUMI_TEST_COVERAGE_PATH/integration-$UUID.cov"
|
|
fi
|
|
- name: Upload code coverage
|
|
uses: actions/upload-artifact@v4
|
|
if: ${{ inputs.enable-coverage }}
|
|
with:
|
|
name: coverage-${{ env.TEST_HASH_ID }}
|
|
path: |
|
|
coverage/*
|
|
!coverage/.gitkeep
|
|
- name: Summarize Test Time by Package
|
|
continue-on-error: true
|
|
env:
|
|
RUNNER_OS: ${{ runner.os }}
|
|
run: |
|
|
mkdir -p test-results
|
|
touch test-results/empty.json # otherwise goteststats fails below when no files match
|
|
# Remove output lines, they make analysis slower & could leak logs:
|
|
if [ "$RUNNER_OS" == "macOS" ]; then
|
|
# It's just another case of BSD sed, GNU sed.
|
|
sed -i '' -e '/"Action":"output"/d' ./test-results/*.json
|
|
else
|
|
sed -i'' -e '/"Action":"output"/d' ./test-results/*.json
|
|
fi
|
|
|
|
goteststats -statistic pkg-time test-results/*.json
|
|
- name: Summarize Test Times by Individual Test
|
|
continue-on-error: true
|
|
run: |
|
|
goteststats -statistic test-time test-results/*.json | head -n 100 || \
|
|
if [[ $? -eq 141 ]]; then true; else exit $?; fi
|
|
- name: Upload artifacts
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: gotestsum-test-results-${{ env.TEST_HASH_ID }}
|
|
path: test-results/*.json
|