Speed up loading ACLs on startup (#3469) #103
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Dendrite | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "**.go" # only execute on changes to go files | |
| - "go.sum" # or dependency updates | |
| - ".github/workflows/**" # or workflow changes | |
| pull_request: | |
| paths: | |
| - "**.go" | |
| - "go.sum" # or dependency updates | |
| - ".github/workflows/**" | |
| release: | |
| types: [published] | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| wasm: | |
| name: WASM build test | |
| timeout-minutes: 5 | |
| runs-on: ubuntu-latest | |
| if: ${{ false }} # disable for now | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: "go.mod" | |
| cache: true | |
| - name: Install Node | |
| uses: actions/setup-node@v2 | |
| with: | |
| node-version: 14 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.npm | |
| key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-node- | |
| - name: Reconfigure Git to use HTTPS auth for repo packages | |
| run: > | |
| git config --global url."https://github.com/".insteadOf | |
| ssh://[email protected]/ | |
| - name: Install test dependencies | |
| working-directory: ./test/wasm | |
| run: npm ci | |
| - name: Test | |
| run: ./test-dendritejs.sh | |
| # Run golangci-lint | |
| lint: | |
| timeout-minutes: 5 | |
| name: Linting | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install libolm | |
| run: sudo apt-get install libolm-dev libolm3 | |
| - name: Install Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: "go.mod" | |
| - name: golangci-lint | |
| uses: golangci/golangci-lint-action@v6 | |
| # run go test with different go versions | |
| test: | |
| timeout-minutes: 10 | |
| name: Unit tests | |
| runs-on: ubuntu-latest | |
| # Service containers to run with `container-job` | |
| services: | |
| # Label used to access the service container | |
| postgres: | |
| # Docker Hub image | |
| image: postgres:13-alpine | |
| # Provide the password for postgres | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: dendrite | |
| ports: | |
| # Maps tcp port 5432 on service container to the host | |
| - 5432:5432 | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install libolm | |
| run: sudo apt-get install libolm-dev libolm3 | |
| - name: Setup go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: "go.mod" | |
| - uses: actions/cache@v4 | |
| # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true | |
| with: | |
| path: | | |
| ~/.cache/go-build | |
| ~/go/pkg/mod | |
| key: ${{ runner.os }}-go-stable-unit-${{ hashFiles('**/go.sum') }} | |
| restore-keys: | | |
| ${{ runner.os }}-go-stable-unit- | |
| - name: Set up gotestfmt | |
| uses: gotesttools/gotestfmt-action@v2 | |
| with: | |
| # Optional: pass GITHUB_TOKEN to avoid rate limiting. | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - run: go test -json -v ./... 2>&1 | gotestfmt -hide all | |
| env: | |
| POSTGRES_HOST: localhost | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: dendrite | |
| # build Dendrite for linux with different architectures and go versions | |
| build: | |
| name: Build for Linux | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| goos: ["linux"] | |
| goarch: ["amd64", "386"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: "go.mod" | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/go-build | |
| ~/go/pkg/mod | |
| key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }} | |
| restore-keys: | | |
| key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}- | |
| - name: Install dependencies x86 | |
| if: ${{ matrix.goarch == '386' }} | |
| run: sudo apt update && sudo apt-get install -y gcc-multilib | |
| - env: | |
| GOOS: ${{ matrix.goos }} | |
| GOARCH: ${{ matrix.goarch }} | |
| CGO_ENABLED: 1 | |
| CGO_CFLAGS: -fno-stack-protector | |
| run: go build -trimpath -v -o "bin/" ./cmd/... | |
| # build for Windows 64-bit | |
| build_windows: | |
| name: Build for Windows | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| goos: ["windows"] | |
| goarch: ["amd64"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: "go.mod" | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/go-build | |
| ~/go/pkg/mod | |
| key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }} | |
| restore-keys: | | |
| key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}- | |
| - name: Install dependencies | |
| run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc | |
| - env: | |
| GOOS: ${{ matrix.goos }} | |
| GOARCH: ${{ matrix.goarch }} | |
| CGO_ENABLED: 1 | |
| CC: "/usr/bin/x86_64-w64-mingw32-gcc" | |
| run: go build -trimpath -v -o "bin/" ./cmd/... | |
| # Dummy step to gate other tests on without repeating the whole list | |
| initial-tests-done: | |
| name: Initial tests passed | |
| needs: [lint, test, build, build_windows] | |
| runs-on: ubuntu-latest | |
| if: ${{ !cancelled() }} # Run this even if prior jobs were skipped | |
| steps: | |
| - name: Check initial tests passed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| jobs: ${{ toJSON(needs) }} | |
| # run go test with different go versions | |
| integration: | |
| timeout-minutes: 20 | |
| needs: initial-tests-done | |
| name: Integration tests | |
| runs-on: ubuntu-latest | |
| # Service containers to run with `container-job` | |
| services: | |
| # Label used to access the service container | |
| postgres: | |
| # Docker Hub image | |
| image: postgres:13-alpine | |
| # Provide the password for postgres | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: dendrite | |
| ports: | |
| # Maps tcp port 5432 on service container to the host | |
| - 5432:5432 | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install libolm | |
| run: sudo apt-get install libolm-dev libolm3 | |
| - name: Setup go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: "go.mod" | |
| - name: Set up gotestfmt | |
| uses: gotesttools/gotestfmt-action@v2 | |
| with: | |
| # Optional: pass GITHUB_TOKEN to avoid rate limiting. | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/go-build | |
| ~/go/pkg/mod | |
| key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }} | |
| restore-keys: | | |
| ${{ runner.os }}-go-stable-test-race- | |
| - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt -hide all | |
| env: | |
| POSTGRES_HOST: localhost | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: dendrite | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| flags: unittests | |
| fail_ci_if_error: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| # run database upgrade tests | |
| upgrade_test: | |
| name: Upgrade tests | |
| timeout-minutes: 20 | |
| needs: initial-tests-done | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: "go.mod" | |
| cache: true | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/go-build | |
| ~/go/pkg/mod | |
| key: ${{ runner.os }}-go-upgrade-test-${{ hashFiles('**/go.sum') }} | |
| restore-keys: | | |
| ${{ runner.os }}-go-upgrade-test- | |
| - name: Docker version | |
| run: docker version | |
| - name: Build upgrade-tests | |
| run: go build ./cmd/dendrite-upgrade-tests | |
| - name: Test upgrade (PostgreSQL) | |
| run: ./dendrite-upgrade-tests -repository=matrix-org/dendrite --head . | |
| - name: Test upgrade (SQLite) | |
| run: ./dendrite-upgrade-tests --sqlite -repository=matrix-org/dendrite --head . | |
| # run database upgrade tests, skipping over one version | |
| upgrade_test_direct: | |
| name: Upgrade tests from HEAD-2 | |
| timeout-minutes: 20 | |
| needs: initial-tests-done | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: "go.mod" | |
| cache: true | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/go-build | |
| ~/go/pkg/mod | |
| key: ${{ runner.os }}-go-upgrade-direct-test-${{ hashFiles('**/go.sum') }} | |
| restore-keys: | | |
| ${{ runner.os }}-go-upgrade-direct-test- | |
| - name: Docker version | |
| run: docker version | |
| - name: Build upgrade-tests | |
| run: go build ./cmd/dendrite-upgrade-tests | |
| - name: Test upgrade (PostgreSQL) | |
| run: ./dendrite-upgrade-tests -direct -from HEAD-2 -repository=matrix-org/dendrite --head . | |
| - name: Test upgrade (SQLite) | |
| run: ./dendrite-upgrade-tests --sqlite -direct -from HEAD-2 -repository=matrix-org/dendrite --head . | |
| # run Sytest in different variations | |
| sytest: | |
| timeout-minutes: 20 | |
| needs: initial-tests-done | |
| name: "Sytest (${{ matrix.label }})" | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - label: SQLite native | |
| - label: SQLite Cgo | |
| cgo: 1 | |
| - label: PostgreSQL | |
| postgres: postgres | |
| container: | |
| image: matrixdotorg/sytest-dendrite | |
| volumes: | |
| - ${{ github.workspace }}:/src | |
| - /root/.cache/go-build:/github/home/.cache/go-build | |
| - /root/.cache/go-mod:/gopath/pkg/mod | |
| env: | |
| POSTGRES: ${{ matrix.postgres && 1}} | |
| SYTEST_BRANCH: ${{ github.head_ref }} | |
| CGO_ENABLED: ${{ matrix.cgo && 1 }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/go-build | |
| /gopath/pkg/mod | |
| key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }} | |
| restore-keys: | | |
| ${{ runner.os }}-go-sytest- | |
| - name: Run Sytest | |
| run: /bootstrap.sh dendrite | |
| working-directory: /src | |
| - name: Summarise results.tap | |
| if: ${{ always() }} | |
| run: /sytest/scripts/tap_to_gha.pl /logs/results.tap | |
| - name: Sytest List Maintenance | |
| if: ${{ always() }} | |
| run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist | |
| continue-on-error: true # not fatal | |
| - name: Are We Synapse Yet? | |
| if: ${{ always() }} | |
| run: /src/are-we-synapse-yet.py /logs/results.tap -v | |
| continue-on-error: true # not fatal | |
| - name: Upload Sytest logs | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} | |
| with: | |
| name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }}) | |
| path: | | |
| /logs/results.tap | |
| /logs/**/*.log* | |
| # run Complement | |
| complement: | |
| name: "Complement (${{ matrix.label }})" | |
| timeout-minutes: 20 | |
| needs: initial-tests-done | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - label: SQLite native | |
| cgo: 0 | |
| - label: SQLite Cgo | |
| cgo: 1 | |
| - label: PostgreSQL | |
| postgres: Postgres | |
| cgo: 0 | |
| steps: | |
| # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement. | |
| # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path | |
| - name: "Set Go Version" | |
| run: | | |
| echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH | |
| echo "~/go/bin" >> $GITHUB_PATH | |
| - name: "Install Complement Dependencies" | |
| # We don't need to install Go because it is included on the Ubuntu 20.04 image: | |
| # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev | |
| go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest | |
| - name: Run actions/checkout@v4 for dendrite | |
| uses: actions/checkout@v4 | |
| with: | |
| path: dendrite | |
| # Attempt to check out the same branch of Complement as the PR. If it | |
| # doesn't exist, fallback to main. | |
| - name: Checkout complement | |
| shell: bash | |
| run: | | |
| mkdir -p complement | |
| # Attempt to use the version of complement which best matches the current | |
| # build. Depending on whether this is a PR or release, etc. we need to | |
| # use different fallbacks. | |
| # | |
| # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF | |
| # for pull requests, otherwise GITHUB_REF). | |
| # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y | |
| # (GITHUB_BASE_REF for pull requests). | |
| # 3. Use the default complement branch ("master"). | |
| for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do | |
| # Skip empty branch names and merge commits. | |
| if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then | |
| continue | |
| fi | |
| (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break | |
| done | |
| # Build initial Dendrite image | |
| - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile . | |
| working-directory: dendrite | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| # Run Complement | |
| - run: | | |
| set -o pipefail && | |
| go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all | |
| shell: bash | |
| name: Run Complement Tests | |
| env: | |
| COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} | |
| COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_ | |
| working-directory: complement | |
| integration-tests-done: | |
| name: Integration tests passed | |
| needs: | |
| [ | |
| initial-tests-done, | |
| upgrade_test, | |
| upgrade_test_direct, | |
| sytest, | |
| complement, | |
| integration, | |
| ] | |
| runs-on: ubuntu-latest | |
| if: ${{ !cancelled() }} # Run this even if prior jobs were skipped | |
| steps: | |
| - name: Check integration tests passed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| jobs: ${{ toJSON(needs) }} | |
| update-docker-images: | |
| name: Update Docker images | |
| permissions: | |
| packages: write | |
| contents: read | |
| security-events: write # To upload Trivy sarif files | |
| if: github.repository == 'element-hq/dendrite' && github.ref_name == 'main' | |
| needs: [integration-tests-done] | |
| uses: element-hq/dendrite/.github/workflows/docker.yml@main | |
| secrets: | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} |