2022-09-02 05:40:13 +00:00
|
|
|
name: CI
|
|
|
|
|
|
|
|
permissions:
|
|
|
|
contents: read
|
2022-11-08 11:13:27 +00:00
|
|
|
id-token: write
|
2022-09-02 05:40:13 +00:00
|
|
|
|
|
|
|
on:
|
|
|
|
workflow_call:
|
|
|
|
inputs:
|
|
|
|
ref:
|
|
|
|
required: true
|
|
|
|
description: "GitHub ref to use"
|
|
|
|
type: string
|
|
|
|
version:
|
|
|
|
required: true
|
|
|
|
description: "Version to produce"
|
|
|
|
type: string
|
2022-09-16 22:04:05 +00:00
|
|
|
build-all-targets:
|
2022-09-14 01:52:21 +00:00
|
|
|
required: false
|
2022-09-16 22:04:05 +00:00
|
|
|
default: true
|
|
|
|
description: "Build targets to produce, false builds only for Linux amd64."
|
|
|
|
type: boolean
|
2023-03-02 22:33:45 +00:00
|
|
|
test-codegen:
|
|
|
|
required: false
|
|
|
|
default: true
|
|
|
|
description: "Whether to run per-language codegen tests."
|
|
|
|
type: boolean
|
2022-09-02 05:40:13 +00:00
|
|
|
lint:
|
|
|
|
required: false
|
|
|
|
default: true
|
|
|
|
description: "Whether to run lints"
|
|
|
|
type: boolean
|
2022-09-14 01:52:21 +00:00
|
|
|
test-version-sets:
|
2022-09-02 05:40:13 +00:00
|
|
|
required: false
|
2022-09-14 01:52:21 +00:00
|
|
|
default: minimum current
|
2022-09-16 22:04:05 +00:00
|
|
|
description: Version sets on which to run integration tests
|
2022-09-02 05:40:13 +00:00
|
|
|
type: string
|
2022-09-16 22:04:05 +00:00
|
|
|
integration-test-platforms:
|
2022-09-02 05:40:13 +00:00
|
|
|
required: false
|
2022-09-16 22:04:05 +00:00
|
|
|
default: ubuntu-latest
|
|
|
|
description: Platforms on which to run integration tests, as a space delimited list
|
|
|
|
type: string
|
2023-01-30 19:30:30 +00:00
|
|
|
acceptance-test-platforms:
|
2022-09-16 22:04:05 +00:00
|
|
|
required: false
|
2023-02-03 19:31:47 +00:00
|
|
|
default: windows-latest macos-latest
|
2022-09-14 01:52:21 +00:00
|
|
|
description: Platforms on which to run integration tests, as a space delimited list
|
2022-09-02 05:40:13 +00:00
|
|
|
type: string
|
2022-09-14 01:52:21 +00:00
|
|
|
enable-coverage:
|
|
|
|
description: "Collects coverage stats; requires cov-enabled builds"
|
|
|
|
default: false
|
2022-09-02 05:40:13 +00:00
|
|
|
required: false
|
2022-09-14 01:52:21 +00:00
|
|
|
type: boolean
|
2022-09-19 21:10:23 +00:00
|
|
|
secrets:
|
|
|
|
PULUMI_BOT_TOKEN:
|
|
|
|
required: true
|
|
|
|
description: "GitHub access token, required to mitigate GitHub rate limits"
|
|
|
|
PULUMI_PROD_ACCESS_TOKEN:
|
|
|
|
required: false
|
|
|
|
description: "Pulumi access token, required to run tests against the service"
|
2023-12-04 02:03:42 +00:00
|
|
|
CODECOV_TOKEN:
|
|
|
|
required: false
|
|
|
|
description: "CodeCov token, required to publish CodeCov coverage data"
|
2024-01-03 14:47:09 +00:00
|
|
|
AZURE_TENANT_ID:
|
|
|
|
required: false
|
|
|
|
description: "Azure tenant ID, required to run tests against Azure"
|
|
|
|
AZURE_CLIENT_ID:
|
|
|
|
required: false
|
|
|
|
description: "Azure client ID, required to run tests against Azure"
|
|
|
|
AZURE_CLIENT_SECRET:
|
|
|
|
required: false
|
|
|
|
description: "Azure clients secret, needs to be rotated before 2025-12-21 (see the pulumi-test user in Azure portal)"
|
2024-01-23 09:41:20 +00:00
|
|
|
AZURE_STORAGE_SAS_TOKEN:
|
|
|
|
required: false
|
|
|
|
description: "Azure storage SAS token, required to run tests against Azure"
|
2024-01-18 19:49:30 +00:00
|
|
|
GCP_SERVICE_ACCOUNT:
|
|
|
|
required: false
|
|
|
|
description: "GCP service account, required to run tests against GCP"
|
2022-09-02 05:40:13 +00:00
|
|
|
|
|
|
|
jobs:
|
2022-09-14 01:52:21 +00:00
|
|
|
matrix:
|
2022-09-02 05:40:13 +00:00
|
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
2022-09-14 01:52:21 +00:00
|
|
|
- uses: actions/checkout@v3
|
|
|
|
with:
|
|
|
|
ref: ${{ inputs.ref }}
|
|
|
|
- name: Configure Go Cache Key
|
|
|
|
env:
|
|
|
|
CACHE_KEY: "matrix-setup"
|
|
|
|
run: echo "$CACHE_KEY" > .gocache.tmp
|
2023-01-30 23:02:53 +00:00
|
|
|
- name: Setup Go Caching
|
|
|
|
uses: actions/setup-go@v3 # only used by gotestsum
|
2022-09-14 01:52:21 +00:00
|
|
|
with:
|
|
|
|
go-version: '>=1.19.0' # decoupled from version sets, only used by gotestsum
|
|
|
|
cache: true
|
|
|
|
cache-dependency-path: |
|
|
|
|
pkg/go.sum
|
|
|
|
.gocache.tmp
|
|
|
|
- uses: actions/checkout@v3
|
|
|
|
with:
|
|
|
|
repository: dnephin/gotestsum
|
|
|
|
ref: d09768c81065b404caed0855eb3ab8f11a2a4431
|
|
|
|
path: vendor/gotestsum
|
|
|
|
- run: |
|
|
|
|
cd vendor/gotestsum
|
|
|
|
go install .
|
|
|
|
- uses: actions/cache@v3
|
|
|
|
with:
|
|
|
|
path: test-results
|
|
|
|
key: read-gotestsum-timing-${{ github.run_number }}
|
|
|
|
restore-keys: gotestsum-timing-
|
|
|
|
- name: build matrix
|
|
|
|
id: matrix
|
|
|
|
env:
|
2022-09-16 22:04:05 +00:00
|
|
|
BUILD_ALL_TARGETS: ${{ inputs.build-all-targets }}
|
2023-03-02 22:33:45 +00:00
|
|
|
TEST_CODEGEN: ${{ inputs.test-codegen }}
|
2022-09-14 01:52:21 +00:00
|
|
|
TEST_VERSION_SETS: ${{ inputs.test-version-sets }}
|
2022-09-16 22:04:05 +00:00
|
|
|
INPUT_INTEGRATION_TEST_PLATFORMS: ${{ inputs.integration-test-platforms }}
|
2023-01-30 19:30:30 +00:00
|
|
|
INPUT_ACCEPTANCE_TEST_PLATFORMS: ${{ inputs.acceptance-test-platforms }}
|
2022-09-14 01:52:21 +00:00
|
|
|
run: |
|
|
|
|
echo "::group::Prime test timing data"
|
|
|
|
mkdir -p test-results
|
|
|
|
find test-results -type f -empty -print -delete || true
|
|
|
|
echo "::endgroup::"
|
2022-09-02 05:40:13 +00:00
|
|
|
|
2022-10-17 08:58:11 +00:00
|
|
|
echo "::group::Remove old test timing data"
|
|
|
|
# Timing data prior to this date is unreliable. Codegen tests modified in #11052 and
|
|
|
|
# merged Monday Oct 17 at 6PM Pacific.
|
|
|
|
find test-results -type f ! -newermt "2022-10-17T18:00-07:00" -print -delete || true
|
|
|
|
echo "::endgroup::"
|
|
|
|
|
2022-09-14 01:52:21 +00:00
|
|
|
echo "::group::Test matrix variables"
|
2022-09-16 22:04:05 +00:00
|
|
|
readarray -td' ' VERSION_SETS_TO_TEST < <(echo -n "$TEST_VERSION_SETS"); declare -p VERSION_SETS_TO_TEST;
|
|
|
|
readarray -td' ' INTEGRATION_PLATFORMS < <(echo -n "$INPUT_INTEGRATION_TEST_PLATFORMS"); declare -p INTEGRATION_PLATFORMS;
|
2023-01-30 19:30:30 +00:00
|
|
|
readarray -td' ' ACCEPTANCE_PLATFORMS < <(echo -n "$INPUT_ACCEPTANCE_TEST_PLATFORMS"); declare -p ACCEPTANCE_PLATFORMS;
|
2022-09-14 01:52:21 +00:00
|
|
|
BUILD_TARGETS='[
|
|
|
|
{ "os": "linux", "arch": "amd64", "build-platform": "ubuntu-latest" }
|
|
|
|
]'
|
2022-09-16 22:04:05 +00:00
|
|
|
if [ "${BUILD_ALL_TARGETS}" = "true" ]; then
|
2022-09-14 01:52:21 +00:00
|
|
|
BUILD_TARGETS='[
|
|
|
|
{ "os": "linux", "arch": "amd64", "build-platform": "ubuntu-latest" },
|
|
|
|
{ "os": "linux", "arch": "arm64", "build-platform": "ubuntu-latest" },
|
|
|
|
{ "os": "windows", "arch": "amd64", "build-platform": "ubuntu-latest" },
|
|
|
|
{ "os": "windows", "arch": "arm64", "build-platform": "ubuntu-latest" },
|
2023-02-02 23:23:54 +00:00
|
|
|
{ "os": "darwin", "arch": "amd64", "build-platform": "ubuntu-latest" },
|
|
|
|
{ "os": "darwin", "arch": "arm64", "build-platform": "ubuntu-latest" }
|
2022-09-14 01:52:21 +00:00
|
|
|
]'
|
|
|
|
fi
|
|
|
|
|
2023-03-02 22:33:45 +00:00
|
|
|
CODEGEN_TESTS_FLAG=--codegen-tests
|
|
|
|
PKG_UNIT_TEST_PARTITIONS=7
|
|
|
|
if [ "${TEST_CODEGEN}" = "false" ]; then
|
|
|
|
CODEGEN_TESTS_FLAG=--no-codegen-tests
|
|
|
|
PKG_UNIT_TEST_PARTITIONS=3
|
|
|
|
fi
|
|
|
|
|
2022-09-16 22:04:05 +00:00
|
|
|
UNIT_TEST_MATRIX=$(
|
|
|
|
./scripts/get-job-matrix.py \
|
|
|
|
-vvv \
|
|
|
|
generate-matrix \
|
|
|
|
--kind unit-test \
|
2023-03-02 22:33:45 +00:00
|
|
|
"$CODEGEN_TESTS_FLAG" \
|
2022-09-16 22:04:05 +00:00
|
|
|
--platform ubuntu-latest \
|
|
|
|
--version-set current \
|
2024-01-25 16:26:55 +00:00
|
|
|
--partition-module cmd/pulumi-test-language 1 \
|
2023-03-02 22:33:45 +00:00
|
|
|
--partition-module pkg "$PKG_UNIT_TEST_PARTITIONS" \
|
2022-09-16 22:04:05 +00:00
|
|
|
--partition-module sdk 1 \
|
2023-06-29 21:48:47 +00:00
|
|
|
--partition-module sdk/go/pulumi-language-go 1 \
|
|
|
|
--partition-module sdk/nodejs/cmd/pulumi-language-nodejs 1 \
|
2023-08-31 16:35:21 +00:00
|
|
|
--partition-module sdk/python/cmd/pulumi-language-python 1 \
|
2022-10-17 05:56:57 +00:00
|
|
|
--partition-module tests 2
|
2022-09-16 22:04:05 +00:00
|
|
|
)
|
2022-09-14 01:52:21 +00:00
|
|
|
|
2022-09-16 22:04:05 +00:00
|
|
|
INTEGRATION_TEST_MATRIX=$(
|
|
|
|
./scripts/get-job-matrix.py \
|
|
|
|
-vvv \
|
|
|
|
generate-matrix \
|
|
|
|
--kind integration-test \
|
2023-03-02 22:33:45 +00:00
|
|
|
"$CODEGEN_TESTS_FLAG" \
|
2022-09-16 22:04:05 +00:00
|
|
|
--platform "${INTEGRATION_PLATFORMS[@]}" \
|
|
|
|
--version-set "${VERSION_SETS_TO_TEST[@]}" \
|
|
|
|
--partition-module pkg 1 \
|
|
|
|
--partition-module sdk 1 \
|
|
|
|
--partition-module tests 2 \
|
|
|
|
--partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8
|
|
|
|
)
|
|
|
|
|
2024-03-02 21:14:07 +00:00
|
|
|
ACCEPTANCE_TEST_MATRIX_WIN="{}"
|
reduce the number of partitions for macos runners (#15243)
I was watching the merge queue quite closely today, and it's continually
stuck waiting to run jobs on MacOS runners. However when the jobs are
running on them they tend to be faster than e.g. Windows runners, often
running the tests assigned to them in 2 min, vs. 8 min on windows.
In addition we're spending quite a bit of time to setup the actual
running of tests, and by spreading the tests out over multiple runners
that time multiplies. Given the macos runners are the most expensive
ones that probably also adds up.
Given all that, we can reduce the number of MacOS runners we're using,
while not trading that off against longer overall CI times, as those are
limited by slower jobs.
Unfortunately this results in some duplication in the ci job, but I
think the tradeoff here might be worth it.
(Also happy to hear if we think this isn't worth the extra messiness in
the GitHub actions files, as usually the merge queue isn't as bad as it
is today, so this isn't that much of an issue)
---------
Co-authored-by: Justin Van Patten <jvp@justinvp.com>
2024-01-29 14:21:05 +00:00
|
|
|
if [[ " ${ACCEPTANCE_PLATFORMS[*]} " =~ [[:space:]]windows-latest[[:space:]] ]]; then
|
|
|
|
ACCEPTANCE_TEST_MATRIX_WIN=$(
|
|
|
|
./scripts/get-job-matrix.py \
|
|
|
|
-vvv \
|
|
|
|
generate-matrix \
|
|
|
|
--kind acceptance-test \
|
|
|
|
"$CODEGEN_TESTS_FLAG" \
|
|
|
|
--tags all xplatform_acceptance \
|
|
|
|
--platform windows-latest \
|
|
|
|
--version-set current \
|
|
|
|
--partition-module pkg 1 \
|
|
|
|
--partition-module sdk 1 \
|
|
|
|
--partition-module tests 1 \
|
|
|
|
--partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8
|
|
|
|
)
|
|
|
|
fi
|
|
|
|
|
2024-03-02 21:14:07 +00:00
|
|
|
ACCEPTANCE_TEST_MATRIX_MACOS="{}"
|
reduce the number of partitions for macos runners (#15243)
I was watching the merge queue quite closely today, and it's continually
stuck waiting to run jobs on MacOS runners. However when the jobs are
running on them they tend to be faster than e.g. Windows runners, often
running the tests assigned to them in 2 min, vs. 8 min on windows.
In addition we're spending quite a bit of time to setup the actual
running of tests, and by spreading the tests out over multiple runners
that time multiplies. Given the macos runners are the most expensive
ones that probably also adds up.
Given all that, we can reduce the number of MacOS runners we're using,
while not trading that off against longer overall CI times, as those are
limited by slower jobs.
Unfortunately this results in some duplication in the ci job, but I
think the tradeoff here might be worth it.
(Also happy to hear if we think this isn't worth the extra messiness in
the GitHub actions files, as usually the merge queue isn't as bad as it
is today, so this isn't that much of an issue)
---------
Co-authored-by: Justin Van Patten <jvp@justinvp.com>
2024-01-29 14:21:05 +00:00
|
|
|
if [[ " ${ACCEPTANCE_PLATFORMS[*]} " =~ [[:space:]]macos-latest[[:space:]] ]]; then
|
|
|
|
ACCEPTANCE_TEST_MATRIX_MACOS=$(
|
|
|
|
./scripts/get-job-matrix.py \
|
|
|
|
-vvv \
|
|
|
|
generate-matrix \
|
|
|
|
--kind acceptance-test \
|
|
|
|
"$CODEGEN_TESTS_FLAG" \
|
|
|
|
--tags all xplatform_acceptance \
|
|
|
|
--platform macos-latest \
|
|
|
|
--version-set current \
|
|
|
|
--partition-module pkg 1 \
|
|
|
|
--partition-module sdk 1 \
|
|
|
|
--partition-module tests 1 \
|
|
|
|
--partition-package github.com/pulumi/pulumi/tests/integration tests/integration 2
|
|
|
|
)
|
|
|
|
fi
|
2022-09-14 01:52:21 +00:00
|
|
|
echo "::endgroup::"
|
2022-09-16 22:04:05 +00:00
|
|
|
|
2022-09-14 01:52:21 +00:00
|
|
|
echo "::group::Version set variable"
|
|
|
|
VERSION_SET=$(./scripts/get-job-matrix.py \
|
|
|
|
generate-version-set \
|
|
|
|
--version-set current
|
|
|
|
)
|
|
|
|
echo "::endgroup::"
|
|
|
|
|
2022-09-16 22:04:05 +00:00
|
|
|
echo "::group::Unit test matrix"
|
|
|
|
echo "$UNIT_TEST_MATRIX" | yq -P '.'
|
|
|
|
echo "::endgroup::"
|
2022-09-14 01:52:21 +00:00
|
|
|
echo "::group::Integration test matrix"
|
|
|
|
echo "$INTEGRATION_TEST_MATRIX" | yq -P '.'
|
|
|
|
echo "::endgroup::"
|
reduce the number of partitions for macos runners (#15243)
I was watching the merge queue quite closely today, and it's continually
stuck waiting to run jobs on MacOS runners. However when the jobs are
running on them they tend to be faster than e.g. Windows runners, often
running the tests assigned to them in 2 min, vs. 8 min on windows.
In addition we're spending quite a bit of time to setup the actual
running of tests, and by spreading the tests out over multiple runners
that time multiplies. Given the macos runners are the most expensive
ones that probably also adds up.
Given all that, we can reduce the number of MacOS runners we're using,
while not trading that off against longer overall CI times, as those are
limited by slower jobs.
Unfortunately this results in some duplication in the ci job, but I
think the tradeoff here might be worth it.
(Also happy to hear if we think this isn't worth the extra messiness in
the GitHub actions files, as usually the merge queue isn't as bad as it
is today, so this isn't that much of an issue)
---------
Co-authored-by: Justin Van Patten <jvp@justinvp.com>
2024-01-29 14:21:05 +00:00
|
|
|
echo "::group::acceptance test matrix windows"
|
|
|
|
echo "$ACCEPTANCE_TEST_MATRIX_WIN" | yq -P '.'
|
|
|
|
echo "::endgroup::"
|
|
|
|
echo "::group::acceptance test matrix macos"
|
|
|
|
echo "$ACCEPTANCE_TEST_MATRIX_MACOS" | yq -P '.'
|
2022-09-14 01:52:21 +00:00
|
|
|
echo "::endgroup::"
|
|
|
|
echo "::group::Version set"
|
|
|
|
echo "$VERSION_SET" | yq -P '.'
|
|
|
|
echo "::endgroup::"
|
|
|
|
|
|
|
|
echo "::group::Set outputs"
|
|
|
|
./.github/scripts/set-output unit-test-matrix "${UNIT_TEST_MATRIX}"
|
2022-09-16 22:04:05 +00:00
|
|
|
./.github/scripts/set-output integration-test-matrix "${INTEGRATION_TEST_MATRIX}"
|
reduce the number of partitions for macos runners (#15243)
I was watching the merge queue quite closely today, and it's continually
stuck waiting to run jobs on MacOS runners. However when the jobs are
running on them they tend to be faster than e.g. Windows runners, often
running the tests assigned to them in 2 min, vs. 8 min on windows.
In addition we're spending quite a bit of time to setup the actual
running of tests, and by spreading the tests out over multiple runners
that time multiplies. Given the macos runners are the most expensive
ones that probably also adds up.
Given all that, we can reduce the number of MacOS runners we're using,
while not trading that off against longer overall CI times, as those are
limited by slower jobs.
Unfortunately this results in some duplication in the ci job, but I
think the tradeoff here might be worth it.
(Also happy to hear if we think this isn't worth the extra messiness in
the GitHub actions files, as usually the merge queue isn't as bad as it
is today, so this isn't that much of an issue)
---------
Co-authored-by: Justin Van Patten <jvp@justinvp.com>
2024-01-29 14:21:05 +00:00
|
|
|
./.github/scripts/set-output acceptance-test-matrix-win "${ACCEPTANCE_TEST_MATRIX_WIN}"
|
|
|
|
./.github/scripts/set-output acceptance-test-matrix-macos "${ACCEPTANCE_TEST_MATRIX_MACOS}"
|
2022-09-14 01:52:21 +00:00
|
|
|
./.github/scripts/set-output version-set "${VERSION_SET}"
|
|
|
|
./.github/scripts/set-output build-targets "${BUILD_TARGETS}"
|
|
|
|
echo "::endgroup::"
|
|
|
|
outputs:
|
|
|
|
unit-test-matrix: "${{ fromJson(steps.matrix.outputs.unit-test-matrix) }}"
|
2022-09-16 22:04:05 +00:00
|
|
|
integration-test-matrix: "${{ fromJson(steps.matrix.outputs.integration-test-matrix) }}"
|
reduce the number of partitions for macos runners (#15243)
I was watching the merge queue quite closely today, and it's continually
stuck waiting to run jobs on MacOS runners. However when the jobs are
running on them they tend to be faster than e.g. Windows runners, often
running the tests assigned to them in 2 min, vs. 8 min on windows.
In addition we're spending quite a bit of time to setup the actual
running of tests, and by spreading the tests out over multiple runners
that time multiplies. Given the macos runners are the most expensive
ones that probably also adds up.
Given all that, we can reduce the number of MacOS runners we're using,
while not trading that off against longer overall CI times, as those are
limited by slower jobs.
Unfortunately this results in some duplication in the ci job, but I
think the tradeoff here might be worth it.
(Also happy to hear if we think this isn't worth the extra messiness in
the GitHub actions files, as usually the merge queue isn't as bad as it
is today, so this isn't that much of an issue)
---------
Co-authored-by: Justin Van Patten <jvp@justinvp.com>
2024-01-29 14:21:05 +00:00
|
|
|
acceptance-test-matrix-win: "${{ fromJson(steps.matrix.outputs.acceptance-test-matrix-win) }}"
|
|
|
|
acceptance-test-matrix-macos: "${{ fromJson(steps.matrix.outputs.acceptance-test-matrix-macos) }}"
|
2022-09-14 01:52:21 +00:00
|
|
|
version-set: "${{ fromJson(steps.matrix.outputs.version-set) }}"
|
|
|
|
build-targets: "${{ fromJson(steps.matrix.outputs.build-targets) }}"
|
|
|
|
|
|
|
|
lint:
|
2023-01-30 18:51:07 +00:00
|
|
|
name: Lint
|
2022-09-14 01:52:21 +00:00
|
|
|
needs: [matrix]
|
|
|
|
if: ${{ inputs.lint }}
|
|
|
|
uses: ./.github/workflows/ci-lint.yml
|
2022-09-02 05:40:13 +00:00
|
|
|
with:
|
|
|
|
ref: ${{ inputs.ref }}
|
2022-09-14 01:52:21 +00:00
|
|
|
version-set: ${{ needs.matrix.outputs.version-set }}
|
2022-09-02 05:40:13 +00:00
|
|
|
|
|
|
|
build-binaries:
|
|
|
|
name: build binaries
|
2022-09-14 01:52:21 +00:00
|
|
|
needs: [matrix]
|
2022-09-02 05:40:13 +00:00
|
|
|
strategy:
|
|
|
|
# To avoid tying up macOS runners:
|
|
|
|
# If using IDE, ignore yaml-schema error: 'Incorrect type. Expected "boolean"'
|
2022-09-14 01:52:21 +00:00
|
|
|
fail-fast: ${{ contains(needs.matrix.outputs.build-targets, 'macos') }}
|
2022-09-02 05:40:13 +00:00
|
|
|
matrix:
|
2022-09-14 01:52:21 +00:00
|
|
|
target: ${{ fromJson(needs.matrix.outputs.build-targets) }}
|
2022-09-02 05:40:13 +00:00
|
|
|
uses: ./.github/workflows/ci-build-binaries.yml
|
|
|
|
with:
|
|
|
|
ref: ${{ inputs.ref }}
|
|
|
|
version: ${{ inputs.version }}
|
|
|
|
os: ${{ matrix.target.os }}
|
|
|
|
arch: ${{ matrix.target.arch }}
|
|
|
|
build-platform: ${{ matrix.target.build-platform }}
|
2022-09-14 01:52:21 +00:00
|
|
|
version-set: ${{ needs.matrix.outputs.version-set }}
|
ci: Track code coverage
**Overview**
This re-enables tracking of code coverage.
For Go, there are two kinds of coverage at play:
unit test and integration test coverage.
Unit tests follow the usual pattern of running
`go test -cover -coverprofile=whatever.cov`.
For integration tests, we use the new integration test profiling support
[added in Go 1.20](https://go.dev/testing/coverage/).
In short, the way it works is:
# Build a coverage instrumented binary:
go build -cover
# Set GOCOVERDIR to a directory and run the integration tests
# that will invoke this coverage-instrumented binary.
GOCOVERDIR=$(pwd)/coverage
go test ./tests
# $GOCOVERDIR will now be filled with coverage data
# from every invocation of the coverage-instrumented binary.
# Combine it into a single coverage file:
go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov
# The resulting file can be uploaded to codecov as-is.
The above replaces the prior, partially working hacks we had in place
to get coverage-instrumented binaries with `go test -c`
and hijacking the TestMain.
**Notable changes**
- TestMain hijacking is deleted from the Pulumi CLI.
We no longer need this to build coverage-instrumented binaries.
- ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH
because the Pulumi binary no longer accepts a test.coverprofile flag.
This information is now in the GOCOVERDIR environment variable.
- We add an `enable-coverage` parameter to the `ci-build-binaries`
workflow to mirror some of the other workflows.
It will produce coverage-instrumented binaries if this is true.
These binaries are then used by `ci-run-test` which will set
`GOCOVERDIR` and merge the coverage results from it.
- Coverage configuration no longer counts tests, testdata,
and Protobuf-generated code against coverage.
- go-wrapper.sh:
Because we're no longer relying on the `go test -c` hack,
this no longer excludes Windows and language providers
from coverage tracking.
- go-test.py and go-wrapper.sh will include pulumi-language-go and
pulumi-language-nodejs in covered packages.
*Other changes*
- go-test.py:
Fixed a bug where `args` parameters added for coverage were ignored.
Note that this change DOES NOT track coverage for calls made to Pulumi
packages by plugins downloaded from external sources,
e.g. provider plugins. Arguably, that's out of scope of coverage
trackcing for the Pulumi repository.
Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
|
|
|
enable-coverage: ${{ inputs.enable-coverage }}
|
2022-09-02 05:40:13 +00:00
|
|
|
secrets: inherit
|
|
|
|
|
|
|
|
build-sdks:
|
2022-09-14 01:52:21 +00:00
|
|
|
name: Build SDKs
|
|
|
|
needs: [matrix]
|
2022-09-02 05:40:13 +00:00
|
|
|
uses: ./.github/workflows/ci-build-sdks.yml
|
|
|
|
with:
|
|
|
|
ref: ${{ inputs.ref }}
|
|
|
|
version: ${{ inputs.version }}
|
2022-09-14 01:52:21 +00:00
|
|
|
version-set: ${{ needs.matrix.outputs.version-set }}
|
|
|
|
secrets: inherit
|
|
|
|
|
|
|
|
# Tests that can run concurrently with builds.
|
|
|
|
unit-test:
|
|
|
|
# By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
|
|
|
|
# appearing in the rendered title of the job name: It changes this:
|
2022-10-26 09:59:16 +00:00
|
|
|
# CI / Unit Test (cd sdk/dotnet && make dotnet_test, cd sdk/dotnet && make dotnet_test, macos-11, mi... / sdk/dotnet dotnet_test on macos-11/current
|
2022-09-14 01:52:21 +00:00
|
|
|
# (See: https://github.com/pulumi/pulumi/runs/8241055084?check_suite_focus=true#logs)
|
|
|
|
# To this:
|
2022-10-26 09:59:16 +00:00
|
|
|
# CI / Unit Test / sdk/dotnet dotnet_test on macos-11/current
|
2022-09-14 01:52:21 +00:00
|
|
|
name: Unit Test${{ matrix.platform && '' }}
|
2023-01-30 18:51:07 +00:00
|
|
|
needs: [matrix, lint]
|
2022-09-16 22:04:05 +00:00
|
|
|
if: ${{ needs.matrix.outputs.unit-test-matrix != '{}' }}
|
2022-09-14 01:52:21 +00:00
|
|
|
strategy:
|
|
|
|
fail-fast: ${{ contains(needs.matrix.outputs.unit-test-matrix, 'macos') }}
|
|
|
|
matrix: ${{ fromJson(needs.matrix.outputs.unit-test-matrix) }}
|
|
|
|
uses: ./.github/workflows/ci-run-test.yml
|
|
|
|
with:
|
|
|
|
ref: ${{ inputs.ref }}
|
|
|
|
version: ${{ inputs.version }}
|
|
|
|
platform: ${{ matrix.platform }}
|
|
|
|
|
|
|
|
test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
|
|
|
|
test-command: ${{ matrix.test-suite.command }}
|
|
|
|
is-integration-test: false
|
ci: Track code coverage
**Overview**
This re-enables tracking of code coverage.
For Go, there are two kinds of coverage at play:
unit test and integration test coverage.
Unit tests follow the usual pattern of running
`go test -cover -coverprofile=whatever.cov`.
For integration tests, we use the new integration test profiling support
[added in Go 1.20](https://go.dev/testing/coverage/).
In short, the way it works is:
# Build a coverage instrumented binary:
go build -cover
# Set GOCOVERDIR to a directory and run the integration tests
# that will invoke this coverage-instrumented binary.
GOCOVERDIR=$(pwd)/coverage
go test ./tests
# $GOCOVERDIR will now be filled with coverage data
# from every invocation of the coverage-instrumented binary.
# Combine it into a single coverage file:
go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov
# The resulting file can be uploaded to codecov as-is.
The above replaces the prior, partially working hacks we had in place
to get coverage-instrumented binaries with `go test -c`
and hijacking the TestMain.
**Notable changes**
- TestMain hijacking is deleted from the Pulumi CLI.
We no longer need this to build coverage-instrumented binaries.
- ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH
because the Pulumi binary no longer accepts a test.coverprofile flag.
This information is now in the GOCOVERDIR environment variable.
- We add an `enable-coverage` parameter to the `ci-build-binaries`
workflow to mirror some of the other workflows.
It will produce coverage-instrumented binaries if this is true.
These binaries are then used by `ci-run-test` which will set
`GOCOVERDIR` and merge the coverage results from it.
- Coverage configuration no longer counts tests, testdata,
and Protobuf-generated code against coverage.
- go-wrapper.sh:
Because we're no longer relying on the `go test -c` hack,
this no longer excludes Windows and language providers
from coverage tracking.
- go-test.py and go-wrapper.sh will include pulumi-language-go and
pulumi-language-nodejs in covered packages.
*Other changes*
- go-test.py:
Fixed a bug where `args` parameters added for coverage were ignored.
Note that this change DOES NOT track coverage for calls made to Pulumi
packages by plugins downloaded from external sources,
e.g. provider plugins. Arguably, that's out of scope of coverage
trackcing for the Pulumi repository.
Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
|
|
|
enable-coverage: ${{ inputs.enable-coverage }}
|
2022-09-14 01:52:21 +00:00
|
|
|
# require-build: false # TODO, remove ${{ matrix.require-build || false }}
|
|
|
|
|
|
|
|
version-set: ${{ toJson(matrix.version-set) }}
|
2022-09-02 05:40:13 +00:00
|
|
|
secrets: inherit
|
|
|
|
|
2022-09-14 01:52:21 +00:00
|
|
|
# Tests that depend on builds
|
2022-09-02 05:40:13 +00:00
|
|
|
integration-test:
|
2022-09-14 01:52:21 +00:00
|
|
|
# By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
|
|
|
|
# appearing in the rendered title of the job name. See: unit test.
|
|
|
|
name: Integration Test${{ matrix.platform && '' }}
|
2023-01-30 18:51:07 +00:00
|
|
|
needs: [matrix, build-binaries, build-sdks, lint]
|
2022-09-16 22:04:05 +00:00
|
|
|
if: ${{ needs.matrix.outputs.integration-test-matrix != '{}' }}
|
2022-09-02 05:40:13 +00:00
|
|
|
strategy:
|
2022-09-14 01:52:21 +00:00
|
|
|
fail-fast: ${{ contains(needs.matrix.outputs.integration-test-matrix, 'macos') }}
|
|
|
|
matrix: ${{ fromJson(needs.matrix.outputs.integration-test-matrix) }}
|
|
|
|
uses: ./.github/workflows/ci-run-test.yml
|
2022-09-02 05:40:13 +00:00
|
|
|
with:
|
|
|
|
ref: ${{ inputs.ref }}
|
|
|
|
version: ${{ inputs.version }}
|
|
|
|
platform: ${{ matrix.platform }}
|
2022-09-14 01:52:21 +00:00
|
|
|
|
|
|
|
test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
|
|
|
|
test-command: ${{ matrix.test-suite.command }}
|
2024-01-30 12:13:22 +00:00
|
|
|
is-integration-test: true
|
ci: Track code coverage
**Overview**
This re-enables tracking of code coverage.
For Go, there are two kinds of coverage at play:
unit test and integration test coverage.
Unit tests follow the usual pattern of running
`go test -cover -coverprofile=whatever.cov`.
For integration tests, we use the new integration test profiling support
[added in Go 1.20](https://go.dev/testing/coverage/).
In short, the way it works is:
# Build a coverage instrumented binary:
go build -cover
# Set GOCOVERDIR to a directory and run the integration tests
# that will invoke this coverage-instrumented binary.
GOCOVERDIR=$(pwd)/coverage
go test ./tests
# $GOCOVERDIR will now be filled with coverage data
# from every invocation of the coverage-instrumented binary.
# Combine it into a single coverage file:
go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov
# The resulting file can be uploaded to codecov as-is.
The above replaces the prior, partially working hacks we had in place
to get coverage-instrumented binaries with `go test -c`
and hijacking the TestMain.
**Notable changes**
- TestMain hijacking is deleted from the Pulumi CLI.
We no longer need this to build coverage-instrumented binaries.
- ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH
because the Pulumi binary no longer accepts a test.coverprofile flag.
This information is now in the GOCOVERDIR environment variable.
- We add an `enable-coverage` parameter to the `ci-build-binaries`
workflow to mirror some of the other workflows.
It will produce coverage-instrumented binaries if this is true.
These binaries are then used by `ci-run-test` which will set
`GOCOVERDIR` and merge the coverage results from it.
- Coverage configuration no longer counts tests, testdata,
and Protobuf-generated code against coverage.
- go-wrapper.sh:
Because we're no longer relying on the `go test -c` hack,
this no longer excludes Windows and language providers
from coverage tracking.
- go-test.py and go-wrapper.sh will include pulumi-language-go and
pulumi-language-nodejs in covered packages.
*Other changes*
- go-test.py:
Fixed a bug where `args` parameters added for coverage were ignored.
Note that this change DOES NOT track coverage for calls made to Pulumi
packages by plugins downloaded from external sources,
e.g. provider plugins. Arguably, that's out of scope of coverage
trackcing for the Pulumi repository.
Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
|
|
|
enable-coverage: ${{ inputs.enable-coverage }}
|
2022-09-14 01:52:21 +00:00
|
|
|
# require-build: false # TODO, remove ${{ matrix.require-build || false }}
|
|
|
|
|
|
|
|
version-set: ${{ toJson(matrix.version-set) }}
|
2022-09-02 05:40:13 +00:00
|
|
|
secrets: inherit
|
2022-09-14 01:52:21 +00:00
|
|
|
|
reduce the number of partitions for macos runners (#15243)
I was watching the merge queue quite closely today, and it's continually
stuck waiting to run jobs on MacOS runners. However when the jobs are
running on them they tend to be faster than e.g. Windows runners, often
running the tests assigned to them in 2 min, vs. 8 min on windows.
In addition we're spending quite a bit of time to setup the actual
running of tests, and by spreading the tests out over multiple runners
that time multiplies. Given the macos runners are the most expensive
ones that probably also adds up.
Given all that, we can reduce the number of MacOS runners we're using,
while not trading that off against longer overall CI times, as those are
limited by slower jobs.
Unfortunately this results in some duplication in the ci job, but I
think the tradeoff here might be worth it.
(Also happy to hear if we think this isn't worth the extra messiness in
the GitHub actions files, as usually the merge queue isn't as bad as it
is today, so this isn't that much of an issue)
---------
Co-authored-by: Justin Van Patten <jvp@justinvp.com>
2024-01-29 14:21:05 +00:00
|
|
|
# Tests that depend on builds, but a smaller subset against Windows platform.
|
|
|
|
acceptance-test-win:
|
|
|
|
# By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
|
|
|
|
# appearing in the rendered title of the job name. See: unit test.
|
|
|
|
name: Acceptance Test${{ matrix.platform && '' }}
|
|
|
|
needs: [matrix, build-binaries, build-sdks]
|
|
|
|
if: ${{ needs.matrix.outputs.acceptance-test-matrix-win != '{}' }}
|
|
|
|
# alow jobs to fail if the platform contains windows
|
|
|
|
strategy:
|
|
|
|
matrix: ${{ fromJson(needs.matrix.outputs.acceptance-test-matrix-win) }}
|
|
|
|
uses: ./.github/workflows/ci-run-test.yml
|
|
|
|
with:
|
|
|
|
ref: ${{ inputs.ref }}
|
|
|
|
version: ${{ inputs.version }}
|
|
|
|
platform: ${{ matrix.platform }}
|
|
|
|
|
|
|
|
test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
|
|
|
|
test-command: ${{ matrix.test-suite.command }}
|
2024-01-30 12:13:22 +00:00
|
|
|
is-integration-test: true
|
reduce the number of partitions for macos runners (#15243)
I was watching the merge queue quite closely today, and it's continually
stuck waiting to run jobs on MacOS runners. However when the jobs are
running on them they tend to be faster than e.g. Windows runners, often
running the tests assigned to them in 2 min, vs. 8 min on windows.
In addition we're spending quite a bit of time to setup the actual
running of tests, and by spreading the tests out over multiple runners
that time multiplies. Given the macos runners are the most expensive
ones that probably also adds up.
Given all that, we can reduce the number of MacOS runners we're using,
while not trading that off against longer overall CI times, as those are
limited by slower jobs.
Unfortunately this results in some duplication in the ci job, but I
think the tradeoff here might be worth it.
(Also happy to hear if we think this isn't worth the extra messiness in
the GitHub actions files, as usually the merge queue isn't as bad as it
is today, so this isn't that much of an issue)
---------
Co-authored-by: Justin Van Patten <jvp@justinvp.com>
2024-01-29 14:21:05 +00:00
|
|
|
enable-coverage: ${{ inputs.enable-coverage }}
|
|
|
|
# require-build: false # TODO, remove ${{ matrix.require-build || false }}
|
|
|
|
|
|
|
|
version-set: ${{ toJson(matrix.version-set) }}
|
|
|
|
secrets: inherit
|
|
|
|
|
|
|
|
# Tests that depend on builds, but a smaller subset against MacOS platform.
|
|
|
|
acceptance-test-macos:
|
2022-09-16 22:04:05 +00:00
|
|
|
# By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
|
|
|
|
# appearing in the rendered title of the job name. See: unit test.
|
2023-01-30 20:39:04 +00:00
|
|
|
name: Acceptance Test${{ matrix.platform && '' }}
|
2022-09-16 22:04:05 +00:00
|
|
|
needs: [matrix, build-binaries, build-sdks]
|
reduce the number of partitions for macos runners (#15243)
I was watching the merge queue quite closely today, and it's continually
stuck waiting to run jobs on MacOS runners. However when the jobs are
running on them they tend to be faster than e.g. Windows runners, often
running the tests assigned to them in 2 min, vs. 8 min on windows.
In addition we're spending quite a bit of time to setup the actual
running of tests, and by spreading the tests out over multiple runners
that time multiplies. Given the macos runners are the most expensive
ones that probably also adds up.
Given all that, we can reduce the number of MacOS runners we're using,
while not trading that off against longer overall CI times, as those are
limited by slower jobs.
Unfortunately this results in some duplication in the ci job, but I
think the tradeoff here might be worth it.
(Also happy to hear if we think this isn't worth the extra messiness in
the GitHub actions files, as usually the merge queue isn't as bad as it
is today, so this isn't that much of an issue)
---------
Co-authored-by: Justin Van Patten <jvp@justinvp.com>
2024-01-29 14:21:05 +00:00
|
|
|
if: ${{ needs.matrix.outputs.acceptance-test-matrix-macos != '{}' }}
|
2022-11-25 21:48:05 +00:00
|
|
|
# alow jobs to fail if the platform contains windows
|
2022-09-16 22:04:05 +00:00
|
|
|
strategy:
|
reduce the number of partitions for macos runners (#15243)
I was watching the merge queue quite closely today, and it's continually
stuck waiting to run jobs on MacOS runners. However when the jobs are
running on them they tend to be faster than e.g. Windows runners, often
running the tests assigned to them in 2 min, vs. 8 min on windows.
In addition we're spending quite a bit of time to setup the actual
running of tests, and by spreading the tests out over multiple runners
that time multiplies. Given the macos runners are the most expensive
ones that probably also adds up.
Given all that, we can reduce the number of MacOS runners we're using,
while not trading that off against longer overall CI times, as those are
limited by slower jobs.
Unfortunately this results in some duplication in the ci job, but I
think the tradeoff here might be worth it.
(Also happy to hear if we think this isn't worth the extra messiness in
the GitHub actions files, as usually the merge queue isn't as bad as it
is today, so this isn't that much of an issue)
---------
Co-authored-by: Justin Van Patten <jvp@justinvp.com>
2024-01-29 14:21:05 +00:00
|
|
|
fail-fast: true
|
|
|
|
matrix: ${{ fromJson(needs.matrix.outputs.acceptance-test-matrix-macos) }}
|
2022-09-16 22:04:05 +00:00
|
|
|
uses: ./.github/workflows/ci-run-test.yml
|
|
|
|
with:
|
|
|
|
ref: ${{ inputs.ref }}
|
|
|
|
version: ${{ inputs.version }}
|
|
|
|
platform: ${{ matrix.platform }}
|
|
|
|
|
|
|
|
test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
|
|
|
|
test-command: ${{ matrix.test-suite.command }}
|
2024-01-30 12:13:22 +00:00
|
|
|
is-integration-test: true
|
ci: Track code coverage
**Overview**
This re-enables tracking of code coverage.
For Go, there are two kinds of coverage at play:
unit test and integration test coverage.
Unit tests follow the usual pattern of running
`go test -cover -coverprofile=whatever.cov`.
For integration tests, we use the new integration test profiling support
[added in Go 1.20](https://go.dev/testing/coverage/).
In short, the way it works is:
# Build a coverage instrumented binary:
go build -cover
# Set GOCOVERDIR to a directory and run the integration tests
# that will invoke this coverage-instrumented binary.
GOCOVERDIR=$(pwd)/coverage
go test ./tests
# $GOCOVERDIR will now be filled with coverage data
# from every invocation of the coverage-instrumented binary.
# Combine it into a single coverage file:
go tool covdata textfmt -i=$(GOCOVERDIR) -o=out.cov
# The resulting file can be uploaded to codecov as-is.
The above replaces the prior, partially working hacks we had in place
to get coverage-instrumented binaries with `go test -c`
and hijacking the TestMain.
**Notable changes**
- TestMain hijacking is deleted from the Pulumi CLI.
We no longer need this to build coverage-instrumented binaries.
- ProgramTest no longer tracks or passes PULUMI_TEST_COVERAGE_PATH
because the Pulumi binary no longer accepts a test.coverprofile flag.
This information is now in the GOCOVERDIR environment variable.
- We add an `enable-coverage` parameter to the `ci-build-binaries`
workflow to mirror some of the other workflows.
It will produce coverage-instrumented binaries if this is true.
These binaries are then used by `ci-run-test` which will set
`GOCOVERDIR` and merge the coverage results from it.
- Coverage configuration no longer counts tests, testdata,
and Protobuf-generated code against coverage.
- go-wrapper.sh:
Because we're no longer relying on the `go test -c` hack,
this no longer excludes Windows and language providers
from coverage tracking.
- go-test.py and go-wrapper.sh will include pulumi-language-go and
pulumi-language-nodejs in covered packages.
*Other changes*
- go-test.py:
Fixed a bug where `args` parameters added for coverage were ignored.
Note that this change DOES NOT track coverage for calls made to Pulumi
packages by plugins downloaded from external sources,
e.g. provider plugins. Arguably, that's out of scope of coverage
trackcing for the Pulumi repository.
Resolves #8615, #11419
2023-06-27 16:57:36 +00:00
|
|
|
enable-coverage: ${{ inputs.enable-coverage }}
|
2022-09-16 22:04:05 +00:00
|
|
|
# require-build: false # TODO, remove ${{ matrix.require-build || false }}
|
|
|
|
|
|
|
|
version-set: ${{ toJson(matrix.version-set) }}
|
|
|
|
secrets: inherit
|
|
|
|
|
2022-09-14 01:52:21 +00:00
|
|
|
test-collect-reports:
|
reduce the number of partitions for macos runners (#15243)
I was watching the merge queue quite closely today, and it's continually
stuck waiting to run jobs on MacOS runners. However when the jobs are
running on them they tend to be faster than e.g. Windows runners, often
running the tests assigned to them in 2 min, vs. 8 min on windows.
In addition we're spending quite a bit of time to setup the actual
running of tests, and by spreading the tests out over multiple runners
that time multiplies. Given the macos runners are the most expensive
ones that probably also adds up.
Given all that, we can reduce the number of MacOS runners we're using,
while not trading that off against longer overall CI times, as those are
limited by slower jobs.
Unfortunately this results in some duplication in the ci job, but I
think the tradeoff here might be worth it.
(Also happy to hear if we think this isn't worth the extra messiness in
the GitHub actions files, as usually the merge queue isn't as bad as it
is today, so this isn't that much of an issue)
---------
Co-authored-by: Justin Van Patten <jvp@justinvp.com>
2024-01-29 14:21:05 +00:00
|
|
|
needs: [unit-test, integration-test, acceptance-test-win, acceptance-test-macos]
|
2022-09-16 22:04:05 +00:00
|
|
|
if: ${{ always() }}
|
2022-09-14 01:52:21 +00:00
|
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
|
|
- uses: actions/cache@v3
|
|
|
|
with:
|
|
|
|
path: test-results
|
|
|
|
key: gotestsum-timing-${{ github.run_number }}
|
|
|
|
restore-keys: gotestsum-timing-
|
|
|
|
- uses: actions/download-artifact@v3
|
|
|
|
continue-on-error: true
|
|
|
|
with:
|
|
|
|
name: gotestsum-test-results
|
|
|
|
path: test-results
|
|
|
|
- name: List and clean up test results
|
|
|
|
continue-on-error: true
|
|
|
|
run: |
|
|
|
|
ls -lhR test-results
|
|
|
|
find test-results -mindepth 1 -name '*.json' -mtime +7 -delete
|
2023-12-04 02:03:42 +00:00
|
|
|
test-collect-coverage:
|
reduce the number of partitions for macos runners (#15243)
I was watching the merge queue quite closely today, and it's continually
stuck waiting to run jobs on MacOS runners. However when the jobs are
running on them they tend to be faster than e.g. Windows runners, often
running the tests assigned to them in 2 min, vs. 8 min on windows.
In addition we're spending quite a bit of time to setup the actual
running of tests, and by spreading the tests out over multiple runners
that time multiplies. Given the macos runners are the most expensive
ones that probably also adds up.
Given all that, we can reduce the number of MacOS runners we're using,
while not trading that off against longer overall CI times, as those are
limited by slower jobs.
Unfortunately this results in some duplication in the ci job, but I
think the tradeoff here might be worth it.
(Also happy to hear if we think this isn't worth the extra messiness in
the GitHub actions files, as usually the merge queue isn't as bad as it
is today, so this isn't that much of an issue)
---------
Co-authored-by: Justin Van Patten <jvp@justinvp.com>
2024-01-29 14:21:05 +00:00
|
|
|
needs: [unit-test, integration-test, acceptance-test-win, acceptance-test-macos]
|
2023-12-04 02:03:42 +00:00
|
|
|
if: ${{ inputs.enable-coverage }}
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
|
|
- name: Retrieve code coverage statuses
|
|
|
|
uses: actions/download-artifact@v3
|
|
|
|
with:
|
|
|
|
path: status
|
|
|
|
# Check that there are no incomplete tests.
|
|
|
|
- name: Check tests completed successfully.
|
|
|
|
run: |
|
|
|
|
if test -z "$(grep -rl 'incomplete' ./status)"; then
|
|
|
|
echo "Tests OK!"
|
|
|
|
else
|
|
|
|
echo "Test Failure.. skipping CodeCov upload."
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
# Checkout repository to upload coverage results.
|
|
|
|
- uses: actions/checkout@v3
|
|
|
|
with:
|
|
|
|
ref: ${{ inputs.ref }}
|
|
|
|
- name: Retrieve code coverage reports
|
|
|
|
uses: actions/download-artifact@v3
|
|
|
|
with:
|
|
|
|
path: coverage
|
|
|
|
- name: Upload code coverage
|
|
|
|
uses: codecov/codecov-action@v3
|
|
|
|
with:
|
|
|
|
directory: coverage/
|
|
|
|
files: "*,!.gitkeep"
|
|
|
|
fail_ci_if_error: false
|
|
|
|
verbose: true
|
|
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
2023-12-12 15:52:54 +00:00
|
|
|
build-release-binaries:
|
|
|
|
# This overwrites the previously built testing binaries with versions without coverage.
|
|
|
|
name: Rebuild binaries
|
reduce the number of partitions for macos runners (#15243)
I was watching the merge queue quite closely today, and it's continually
stuck waiting to run jobs on MacOS runners. However when the jobs are
running on them they tend to be faster than e.g. Windows runners, often
running the tests assigned to them in 2 min, vs. 8 min on windows.
In addition we're spending quite a bit of time to setup the actual
running of tests, and by spreading the tests out over multiple runners
that time multiplies. Given the macos runners are the most expensive
ones that probably also adds up.
Given all that, we can reduce the number of MacOS runners we're using,
while not trading that off against longer overall CI times, as those are
limited by slower jobs.
Unfortunately this results in some duplication in the ci job, but I
think the tradeoff here might be worth it.
(Also happy to hear if we think this isn't worth the extra messiness in
the GitHub actions files, as usually the merge queue isn't as bad as it
is today, so this isn't that much of an issue)
---------
Co-authored-by: Justin Van Patten <jvp@justinvp.com>
2024-01-29 14:21:05 +00:00
|
|
|
needs: [matrix, unit-test, integration-test, acceptance-test-win, acceptance-test-macos]
|
2023-12-12 15:52:54 +00:00
|
|
|
if: ${{ inputs.enable-coverage }}
|
|
|
|
strategy:
|
|
|
|
# To avoid tying up macOS runners:
|
|
|
|
# If using IDE, ignore yaml-schema error: 'Incorrect type. Expected "boolean"'
|
|
|
|
fail-fast: ${{ contains(needs.matrix.outputs.build-targets, 'macos') }}
|
|
|
|
matrix:
|
|
|
|
target: ${{ fromJson(needs.matrix.outputs.build-targets) }}
|
|
|
|
uses: ./.github/workflows/ci-build-binaries.yml
|
|
|
|
with:
|
|
|
|
ref: ${{ inputs.ref }}
|
|
|
|
version: ${{ inputs.version }}
|
|
|
|
os: ${{ matrix.target.os }}
|
|
|
|
arch: ${{ matrix.target.arch }}
|
|
|
|
build-platform: ${{ matrix.target.build-platform }}
|
|
|
|
version-set: ${{ needs.matrix.outputs.version-set }}
|
|
|
|
enable-coverage: false
|
|
|
|
secrets: inherit
|