498 lines
16 KiB
YAML
498 lines
16 KiB
YAML
name: Dendrite
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
paths:
|
|
- '**.go' # only execute on changes to go files
|
|
- 'go.sum' # or dependency updates
|
|
- '.github/workflows/**' # or workflow changes
|
|
pull_request:
|
|
paths:
|
|
- '**.go'
|
|
- 'go.sum' # or dependency updates
|
|
- '.github/workflows/**'
|
|
release:
|
|
types: [published]
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
wasm:
|
|
name: WASM build test
|
|
timeout-minutes: 5
|
|
runs-on: ubuntu-latest
|
|
if: ${{ false }} # disable for now
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Install Go
|
|
uses: actions/setup-go@v4
|
|
with:
|
|
go-version: "stable"
|
|
cache: true
|
|
|
|
- name: Install Node
|
|
uses: actions/setup-node@v2
|
|
with:
|
|
node-version: 14
|
|
|
|
- uses: actions/cache@v4
|
|
with:
|
|
path: ~/.npm
|
|
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-node-
|
|
|
|
- name: Reconfigure Git to use HTTPS auth for repo packages
|
|
run: >
|
|
git config --global url."https://github.com/".insteadOf
|
|
ssh://git@github.com/
|
|
|
|
- name: Install test dependencies
|
|
working-directory: ./test/wasm
|
|
run: npm ci
|
|
|
|
- name: Test
|
|
run: ./test-dendritejs.sh
|
|
|
|
# Run golangci-lint
|
|
lint:
|
|
timeout-minutes: 5
|
|
name: Linting
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Install libolm
|
|
run: sudo apt-get install libolm-dev libolm3
|
|
- name: Install Go
|
|
uses: actions/setup-go@v4
|
|
with:
|
|
go-version: "stable"
|
|
- name: golangci-lint
|
|
uses: golangci/golangci-lint-action@v3
|
|
|
|
# run go test with different go versions
|
|
test:
|
|
timeout-minutes: 10
|
|
name: Unit tests
|
|
runs-on: ubuntu-latest
|
|
# Service containers to run with `container-job`
|
|
services:
|
|
# Label used to access the service container
|
|
postgres:
|
|
# Docker Hub image
|
|
image: postgres:13-alpine
|
|
# Provide the password for postgres
|
|
env:
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: dendrite
|
|
ports:
|
|
# Maps tcp port 5432 on service container to the host
|
|
- 5432:5432
|
|
# Set health checks to wait until postgres has started
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Install libolm
|
|
run: sudo apt-get install libolm-dev libolm3
|
|
- name: Setup go
|
|
uses: actions/setup-go@v4
|
|
with:
|
|
go-version: "stable"
|
|
- uses: actions/cache@v4
|
|
# manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
|
|
with:
|
|
path: |
|
|
~/.cache/go-build
|
|
~/go/pkg/mod
|
|
key: ${{ runner.os }}-go-stable-unit-${{ hashFiles('**/go.sum') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-go-stable-unit-
|
|
- name: Set up gotestfmt
|
|
uses: gotesttools/gotestfmt-action@v2
|
|
with:
|
|
# Optional: pass GITHUB_TOKEN to avoid rate limiting.
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
- run: go test -json -v ./... 2>&1 | gotestfmt -hide all
|
|
env:
|
|
POSTGRES_HOST: localhost
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: dendrite
|
|
|
|
# build Dendrite for linux with different architectures and go versions
|
|
build:
|
|
name: Build for Linux
|
|
timeout-minutes: 10
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
goos: ["linux"]
|
|
goarch: ["amd64", "386"]
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup go
|
|
uses: actions/setup-go@v4
|
|
with:
|
|
go-version: "stable"
|
|
- uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.cache/go-build
|
|
~/go/pkg/mod
|
|
key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
|
|
restore-keys: |
|
|
key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
|
|
- name: Install dependencies x86
|
|
if: ${{ matrix.goarch == '386' }}
|
|
run: sudo apt update && sudo apt-get install -y gcc-multilib
|
|
- env:
|
|
GOOS: ${{ matrix.goos }}
|
|
GOARCH: ${{ matrix.goarch }}
|
|
CGO_ENABLED: 1
|
|
CGO_CFLAGS: -fno-stack-protector
|
|
run: go build -trimpath -v -o "bin/" ./cmd/...
|
|
|
|
# build for Windows 64-bit
|
|
build_windows:
|
|
name: Build for Windows
|
|
timeout-minutes: 10
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
goos: ["windows"]
|
|
goarch: ["amd64"]
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup Go
|
|
uses: actions/setup-go@v4
|
|
with:
|
|
go-version: "stable"
|
|
- uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.cache/go-build
|
|
~/go/pkg/mod
|
|
key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
|
|
restore-keys: |
|
|
key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
|
|
- name: Install dependencies
|
|
run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
|
|
- env:
|
|
GOOS: ${{ matrix.goos }}
|
|
GOARCH: ${{ matrix.goarch }}
|
|
CGO_ENABLED: 1
|
|
CC: "/usr/bin/x86_64-w64-mingw32-gcc"
|
|
run: go build -trimpath -v -o "bin/" ./cmd/...
|
|
|
|
# Dummy step to gate other tests on without repeating the whole list
|
|
initial-tests-done:
|
|
name: Initial tests passed
|
|
needs: [lint, test, build, build_windows]
|
|
runs-on: ubuntu-latest
|
|
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
|
|
steps:
|
|
- name: Check initial tests passed
|
|
uses: re-actors/alls-green@release/v1
|
|
with:
|
|
jobs: ${{ toJSON(needs) }}
|
|
|
|
# run go test with different go versions
|
|
integration:
|
|
timeout-minutes: 20
|
|
needs: initial-tests-done
|
|
name: Integration tests
|
|
runs-on: ubuntu-latest
|
|
# Service containers to run with `container-job`
|
|
services:
|
|
# Label used to access the service container
|
|
postgres:
|
|
# Docker Hub image
|
|
image: postgres:13-alpine
|
|
# Provide the password for postgres
|
|
env:
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: dendrite
|
|
ports:
|
|
# Maps tcp port 5432 on service container to the host
|
|
- 5432:5432
|
|
# Set health checks to wait until postgres has started
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Install libolm
|
|
run: sudo apt-get install libolm-dev libolm3
|
|
- name: Setup go
|
|
uses: actions/setup-go@v4
|
|
with:
|
|
go-version: "stable"
|
|
- name: Set up gotestfmt
|
|
uses: gotesttools/gotestfmt-action@v2
|
|
with:
|
|
# Optional: pass GITHUB_TOKEN to avoid rate limiting.
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
- uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.cache/go-build
|
|
~/go/pkg/mod
|
|
key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-go-stable-test-race-
|
|
- run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt -hide all
|
|
env:
|
|
POSTGRES_HOST: localhost
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: dendrite
|
|
- name: Upload coverage to Codecov
|
|
uses: codecov/codecov-action@v4
|
|
with:
|
|
flags: unittests
|
|
fail_ci_if_error: true
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
|
|
# run database upgrade tests
|
|
upgrade_test:
|
|
name: Upgrade tests
|
|
timeout-minutes: 20
|
|
needs: initial-tests-done
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup go
|
|
uses: actions/setup-go@v4
|
|
with:
|
|
go-version: "stable"
|
|
cache: true
|
|
- uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.cache/go-build
|
|
~/go/pkg/mod
|
|
key: ${{ runner.os }}-go-upgrade-test-${{ hashFiles('**/go.sum') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-go-upgrade-test-
|
|
- name: Docker version
|
|
run: docker version
|
|
- name: Build upgrade-tests
|
|
run: go build ./cmd/dendrite-upgrade-tests
|
|
- name: Test upgrade (PostgreSQL)
|
|
run: ./dendrite-upgrade-tests --head .
|
|
- name: Test upgrade (SQLite)
|
|
run: ./dendrite-upgrade-tests --sqlite --head .
|
|
|
|
# run database upgrade tests, skipping over one version
|
|
upgrade_test_direct:
|
|
name: Upgrade tests from HEAD-2
|
|
timeout-minutes: 20
|
|
needs: initial-tests-done
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup go
|
|
uses: actions/setup-go@v4
|
|
with:
|
|
go-version: "stable"
|
|
cache: true
|
|
- uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.cache/go-build
|
|
~/go/pkg/mod
|
|
key: ${{ runner.os }}-go-upgrade-direct-test-${{ hashFiles('**/go.sum') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-go-upgrade-direct-test-
|
|
- name: Docker version
|
|
run: docker version
|
|
- name: Build upgrade-tests
|
|
run: go build ./cmd/dendrite-upgrade-tests
|
|
- name: Test upgrade (PostgreSQL)
|
|
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
|
|
- name: Test upgrade (SQLite)
|
|
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
|
|
|
|
# run Sytest in different variations
|
|
sytest:
|
|
timeout-minutes: 20
|
|
needs: initial-tests-done
|
|
name: "Sytest (${{ matrix.label }})"
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- label: SQLite native
|
|
|
|
- label: SQLite Cgo
|
|
cgo: 1
|
|
|
|
- label: PostgreSQL
|
|
postgres: postgres
|
|
|
|
container:
|
|
image: matrixdotorg/sytest-dendrite
|
|
volumes:
|
|
- ${{ github.workspace }}:/src
|
|
- /root/.cache/go-build:/github/home/.cache/go-build
|
|
- /root/.cache/go-mod:/gopath/pkg/mod
|
|
env:
|
|
POSTGRES: ${{ matrix.postgres && 1}}
|
|
SYTEST_BRANCH: ${{ github.head_ref }}
|
|
CGO_ENABLED: ${{ matrix.cgo && 1 }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.cache/go-build
|
|
/gopath/pkg/mod
|
|
key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-go-sytest-
|
|
- name: Run Sytest
|
|
run: /bootstrap.sh dendrite
|
|
working-directory: /src
|
|
- name: Summarise results.tap
|
|
if: ${{ always() }}
|
|
run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
|
|
- name: Sytest List Maintenance
|
|
if: ${{ always() }}
|
|
run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist
|
|
continue-on-error: true # not fatal
|
|
- name: Are We Synapse Yet?
|
|
if: ${{ always() }}
|
|
run: /src/are-we-synapse-yet.py /logs/results.tap -v
|
|
continue-on-error: true # not fatal
|
|
- name: Upload Sytest logs
|
|
uses: actions/upload-artifact@v4
|
|
if: ${{ always() }}
|
|
with:
|
|
name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
|
|
path: |
|
|
/logs/results.tap
|
|
/logs/**/*.log*
|
|
|
|
# run Complement
|
|
complement:
|
|
name: "Complement (${{ matrix.label }})"
|
|
timeout-minutes: 20
|
|
needs: initial-tests-done
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- label: SQLite native
|
|
cgo: 0
|
|
|
|
- label: SQLite Cgo
|
|
cgo: 1
|
|
|
|
- label: PostgreSQL
|
|
postgres: Postgres
|
|
cgo: 0
|
|
steps:
|
|
# Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
|
|
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
|
|
- name: "Set Go Version"
|
|
run: |
|
|
echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
|
|
echo "~/go/bin" >> $GITHUB_PATH
|
|
- name: "Install Complement Dependencies"
|
|
# We don't need to install Go because it is included on the Ubuntu 20.04 image:
|
|
# See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
|
|
run: |
|
|
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
|
|
go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
|
|
- name: Run actions/checkout@v4 for dendrite
|
|
uses: actions/checkout@v4
|
|
with:
|
|
path: dendrite
|
|
|
|
# Attempt to check out the same branch of Complement as the PR. If it
|
|
# doesn't exist, fallback to main.
|
|
- name: Checkout complement
|
|
shell: bash
|
|
run: |
|
|
mkdir -p complement
|
|
# Attempt to use the version of complement which best matches the current
|
|
# build. Depending on whether this is a PR or release, etc. we need to
|
|
# use different fallbacks.
|
|
#
|
|
# 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
|
|
# for pull requests, otherwise GITHUB_REF).
|
|
# 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
|
|
# (GITHUB_BASE_REF for pull requests).
|
|
# 3. Use the default complement branch ("master").
|
|
for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
|
|
# Skip empty branch names and merge commits.
|
|
if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
|
|
continue
|
|
fi
|
|
(wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
|
|
done
|
|
# Build initial Dendrite image
|
|
- run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
|
|
working-directory: dendrite
|
|
env:
|
|
DOCKER_BUILDKIT: 1
|
|
|
|
# Run Complement
|
|
- run: |
|
|
set -o pipefail &&
|
|
go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
|
|
shell: bash
|
|
name: Run Complement Tests
|
|
env:
|
|
COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }}
|
|
COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
|
|
working-directory: complement
|
|
|
|
integration-tests-done:
|
|
name: Integration tests passed
|
|
needs:
|
|
[
|
|
initial-tests-done,
|
|
upgrade_test,
|
|
upgrade_test_direct,
|
|
sytest,
|
|
complement,
|
|
integration
|
|
]
|
|
runs-on: ubuntu-latest
|
|
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
|
|
steps:
|
|
- name: Check integration tests passed
|
|
uses: re-actors/alls-green@release/v1
|
|
with:
|
|
jobs: ${{ toJSON(needs) }}
|
|
|
|
update-docker-images:
|
|
name: Update Docker images
|
|
permissions:
|
|
packages: write
|
|
contents: read
|
|
security-events: write # To upload Trivy sarif files
|
|
if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main'
|
|
needs: [integration-tests-done]
|
|
uses: matrix-org/dendrite/.github/workflows/docker.yml@main
|
|
secrets:
|
|
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
|