pulumi/scripts/retry

49 lines
1.4 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
# This script implements a general retry mechanism for a command. On each iteration, parallelism
# flags are halved and the command is retried.
retries="${PULUMI_TEST_RETRIES:-"0"}"
attempts=1
success=false
retried=false
run_tests() {
export GO_TEST_PARALLELISM="${GO_TEST_PARALLELISM:-"8"}"
export GO_TEST_PKG_PARALLELISM="${GO_TEST_PKG_PARALLELISM:-"2"}"
# TODO: https://github.com/pulumi/pulumi/issues/10699
# Enable running tests with -shuffle on.
export GO_TEST_SHUFFLE=${GO_TEST_SHUFFLE:-"off"}
echo "COMMAND = " "${@}"
until "${@}"; do
retried=true
if [ "${attempts}" -gt "${retries}" ]; then
echo "::warning Failed after ${attempts} attempts"
return
else
echo "::warning Retrying command"
fi
attempts=$((attempts + 1))
export GO_TEST_PARALLELISM=$((GO_TEST_PARALLELISM <= 2 ? 1 : GO_TEST_PARALLELISM / 2))
export GO_TEST_PKG_PARALLELISM=$((GO_TEST_PKG_PARALLELISM <= 2 ? 1 : GO_TEST_PKG_PARALLELISM / 2))
export GO_TEST_SHUFFLE="off"
done
success=true
echo "::info Tests successful.. failures refer to test flakes that were successfully re-run."
}
run_tests "${@}"
echo "TEST_SUCCESS=${success}" >> "$GITHUB_OUTPUT"
echo "TEST_RETRIED=${retried}" >> "$GITHUB_OUTPUT"
if ! "${success}"; then
exit 1
fi