From 0366ef59008692b7a8106d9a1f34d730bde87540 Mon Sep 17 00:00:00 2001 From: Wim Looman Date: Wed, 15 Nov 2023 15:53:34 +0100 Subject: [PATCH] Run separate web server, registry watcher and build servers in docker-compose --- .github/workflows/ci.yml | 48 ++---------- .github/workflows/docker.yml | 6 +- .gitignore | 1 + README.md | 32 ++++---- docker-compose.yml | 147 ++++++++++++++++++++++++++--------- dockerfiles/Dockerfile | 86 ++++++++++++++------ dockerfiles/entrypoint.sh | 38 --------- 7 files changed, 202 insertions(+), 156 deletions(-) delete mode 100755 dockerfiles/entrypoint.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4c1e2fc1..af9388035 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,13 +38,8 @@ jobs: - name: Launch postgres run: | - cp .env.sample .env - mkdir -p ${DOCSRS_PREFIX}/public-html - docker compose up -d db - # Give the database enough time to start up - sleep 5 - # Make sure the database is actually working - psql "${DOCSRS_DATABASE_URL}" + touch .docker.env + docker compose up --wait --wait-timeout 30 db - name: run database migrations run: cargo run -- database migrate @@ -60,9 +55,6 @@ jobs: --check \ -- --all-targets --all-features - - name: Clean up the database - run: docker compose down --volumes - build: runs-on: ubuntu-latest steps: @@ -110,13 +102,8 @@ jobs: - name: Launch postgres and min.io run: | - cp .env.sample .env - mkdir -p ${DOCSRS_PREFIX}/public-html - docker compose up -d db s3 - # Give the database enough time to start up - sleep 5 - # Make sure the database is actually working - psql "${DOCSRS_DATABASE_URL}" + touch .docker.env + docker compose up --wait --wait-timeout 30 db s3 - name: run tests shell: bash @@ -127,9 +114,6 @@ jobs: $f || exit 1 done - - name: Clean up the database - run: docker compose down --volumes - GUI_test: runs-on: ubuntu-latest needs: build @@ -145,20 +129,12 @@ jobs: - name: Launch postgres and min.io run: | - cp .env.sample .env - mkdir -p ${DOCSRS_PREFIX}/public-html - docker compose up -d db s3 - # Give the database enough time to start up - sleep 5 - # Make sure the database is actually working - psql "${DOCSRS_DATABASE_URL}" + touch .docker.env + docker compose up --wait --wait-timeout 30 db s3 - name: Run GUI tests run: ./dockerfiles/run-gui-tests.sh - - name: Clean up the database - run: docker compose down --volumes - build_tests: runs-on: ubuntu-latest needs: build @@ -174,13 +150,8 @@ jobs: - name: Launch postgres and min.io run: | - cp .env.sample .env - mkdir -p ${DOCSRS_PREFIX}/public-html - docker compose up -d db s3 - # Give the database enough time to start up - sleep 5 - # Make sure the database is actually working - psql "${DOCSRS_DATABASE_URL}" + touch .docker.env + docker compose up --wait --wait-timeout 30 db s3 - name: slow tests env: @@ -193,9 +164,6 @@ jobs: $f --ignored --test-threads=1 || exit 1 done - - name: Clean up the database - run: docker compose down --volumes - fmt: name: Rustfmt runs-on: ubuntu-latest diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 84ae05b95..1fd4cb261 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,5 +9,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Build the Docker image - run: docker build -t docs-rs -f dockerfiles/Dockerfile . + - run: docker build --target web-server -f dockerfiles/Dockerfile . + - run: docker build --target build-server -f dockerfiles/Dockerfile . + - run: docker build --target registry-watcher -f dockerfiles/Dockerfile . + - run: docker build --target cli -f dockerfiles/Dockerfile . diff --git a/.gitignore b/.gitignore index 99643b163..9d9ef06ba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /ignored /.env +/.docker.env /src/web/badge/Cargo.lock target *.css diff --git a/README.md b/README.md index 5b2fbbd57..0a1218b1f 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ mkdir -p ignored/cratesfyi-prefix/crates.io-index # Builds the docs.rs binary cargo build # Start the external services. -docker compose up -d db s3 +docker compose up --wait db s3 # anything that doesn't run via docker-compose needs the settings defined in # .env. Either via `. ./.env` as below, or via any dotenv shell integration. . ./.env @@ -115,21 +115,25 @@ which uses docker-compose for the web server as well. This will not cache dependencies - in particular, you'll have to rebuild all 400 whenever the lockfile changes - but makes sure that you're in a known environment so you should have fewer problems getting started. -You can also use the `web` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows): +You'll need to `touch .docker.env` first, this file can have any environment +variable overrides you want to use in docker containers. + +You can also use the `builder-a` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows): + ```sh +# update the toolchain +docker compose run --rm builder-a build update-toolchain # run a build for a single crate -docker compose run web build crate regex 1.3.1 -# or build essential files -docker compose run web build add-essential-files -# rebuild the web container when you changed code. -docker compose up -d web --build +docker compose run --rm builder-a build crate regex 1.3.1 +# rebuild containers when you changed code. +docker compose up --wait --build ``` -You can also run other commands like the setup above from within the container: +You can also run other non-build commands like the setup steps above, or queueing crates for the background builders from within the `cli` container: ```sh docker compose run --rm cli database migrate -docker compose run --rm cli build update-toolchain +docker compose run --rm cli queue add regex 1.3.1 ``` Note that running tests is not supported when using pure docker-compose. @@ -151,7 +155,7 @@ Three services are defined: #### Rebuilding Containers -To rebuild the site, run `docker compose build`. +To rebuild the site, run `docker compose --profile all build`. Note that docker-compose caches the build even if you change the source code, so this will be necessary anytime you make changes. @@ -172,7 +176,7 @@ This is probably because you have `git.autocrlf` set to true, ##### I see the error `/opt/rustwide/cargo-home/bin/cargo: cannot execute binary file: Exec format error` when running builds. -You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker compose run web build [...]` workaround described above. +You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker compose run --rm builder-a build [...]` workaround described above. See [rustwide#41](https://github.com/rust-lang/rustwide/issues/41) for more details about supporting more platforms directly. @@ -200,11 +204,11 @@ cargo run -- start-web-server ```sh # Builds and adds it into database # This is the main command to build and add a documentation into docs.rs. -# For example, `docker compose run web build crate regex 1.1.6` +# For example, `docker compose run --rm builder-a build crate regex 1.1.6` cargo run -- build crate -# alternatively, via the web container -docker compose run web build crate +# alternatively, within docker-compose containers +docker compose run --rm builder-a build crate # Builds every crate on crates.io and adds them into database # (beware: this may take months to finish) diff --git a/docker-compose.yml b/docker-compose.yml index 2dae8a11b..ccdacd62a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,35 +1,118 @@ -version: "3" +version: "3.4" + +x-healthcheck: &healthcheck-interval + interval: 1s + timeout: 1s + start_period: 10s + # TODO: https://github.com/docker/compose/issues/10461 + # interval: 10s + # start_interval: 1s + +x-environment: &environment + RUST_BACKTRACE: true + + DOCSRS_PREFIX: /opt/docsrs/prefix + + DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db + DOCSRS_MIN_POOL_SIZE: 2 + DOCSRS_MAX_POOL_SIZE: 10 + + DOCSRS_STORAGE_BACKEND: s3 + + S3_ENDPOINT: http://s3:9000 + AWS_ACCESS_KEY_ID: cratesfyi + AWS_SECRET_ACCESS_KEY: secret_key + + DOCSRS_RENDER_THREADS: 2 + + DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide + DOCSRS_DOCKER: true + DOCSRS_DOCKER_IMAGE: ghcr.io/rust-lang/crates-build-env/linux-micro + DOCSRS_BUILD_CPU_LIMIT: 2 + DOCSRS_INCLUDE_DEFAULT_TARGETS: false + +x-builder: &builder + build: + context: . + dockerfile: ./dockerfiles/Dockerfile + target: build-server + depends_on: + - db + - s3 + environment: *environment + env_file: + - .docker.env + healthcheck: + << : *healthcheck-interval + test: curl --silent --fail localhost:3000/about/metrics + services: web: build: context: . dockerfile: ./dockerfiles/Dockerfile + target: web-server platform: "linux/amd64" depends_on: - db - s3 ports: - - "3000:3000" - # for metrics - expose: ["3000"] + - "3000:80" + environment: *environment + env_file: + - .docker.env + healthcheck: + << : *healthcheck-interval + test: curl --silent --fail localhost:80/about/metrics + + # Include the registry watcher with `docker compose --profile watch up -d` + registry-watcher: + build: + context: . + dockerfile: ./dockerfiles/Dockerfile + target: registry-watcher + platform: "linux/amd64" + depends_on: + - db volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - - ".rustwide-docker:/opt/docsrs/rustwide" - "cratesio-index:/opt/docsrs/prefix/crates.io-index" - environment: - DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide - DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db - DOCSRS_STORAGE_BACKEND: s3 - S3_ENDPOINT: http://s3:9000 - AWS_ACCESS_KEY_ID: cratesfyi - AWS_SECRET_ACCESS_KEY: secret_key + environment: *environment env_file: - - .env + - .docker.env + profiles: + - watch + - all healthcheck: - test: ["CMD", "curl", "--silent", "--fail", "localhost:3000"] - interval: 10s - timeout: 5s - retries: 10 + << : *healthcheck-interval + test: curl --silent --fail localhost:3000/about/metrics + + builder-a: + << : *builder + volumes: + - ".rustwide-docker/builder-a:/opt/docsrs/rustwide" + - "/var/run/docker.sock:/var/run/docker.sock" + + builder-b: + << : *builder + volumes: + - ".rustwide-docker/builder-b:/opt/docsrs/rustwide" + - "/var/run/docker.sock:/var/run/docker.sock" + + cli: + build: + context: . + dockerfile: ./dockerfiles/Dockerfile + target: cli + depends_on: + - db + - s3 + volumes: + - ".rustwide-docker/cli:/opt/docsrs/rustwide" + environment: *environment + env_file: + - .docker.env + profiles: + - all db: build: @@ -44,10 +127,8 @@ services: # Use a non-standard port on the host to avoid conflicting with existing postgres servers - "127.0.0.1:15432:5432" healthcheck: - test: ["CMD", "pg_isready", "--username", "cratesfyi"] - interval: 10s - timeout: 5s - retries: 10 + << : *healthcheck-interval + test: pg_isready --username cratesfyi s3: image: minio/minio @@ -65,17 +146,8 @@ services: MINIO_ROOT_USER: cratesfyi MINIO_ROOT_PASSWORD: secret_key healthcheck: - test: - [ - "CMD", - "curl", - "--silent", - "--fail", - "localhost:9000/minio/health/ready", - ] - interval: 10s - timeout: 5s - retries: 10 + << : *healthcheck-interval + test: mc ready local prometheus: build: @@ -84,11 +156,8 @@ services: ports: - "127.0.0.1:9090:9090" healthcheck: - test: - ["CMD", "curl", "--silent", "--fail", "localhost:9090/-/ready"] - interval: 10s - timeout: 5s - retries: 10 + << : *healthcheck-interval + test: promtool check healthy gui_tests: build: @@ -99,6 +168,8 @@ services: - "host.docker.internal:host-gateway" volumes: - "${PWD}:/build/out" + profiles: + - all volumes: postgres-data: {} diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 8e0f6d499..fd675a9df 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -65,42 +65,80 @@ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ && DEBIAN_FRONTEND=noninteractive apt-get install -y \ ca-certificates \ + curl \ tini \ && rm -rf /var/lib/apt/lists/* +WORKDIR /srv/docsrs + +# Tini is a small init binary to properly handle signals +ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] +CMD ["start-web-server", "0.0.0.0:80"] + COPY --from=build /build/target/release/cratesfyi /usr/local/bin COPY static /srv/docsrs/static COPY templates /srv/docsrs/templates COPY vendor /srv/docsrs/vendor -WORKDIR /srv/docsrs +######################## +# Build server stage # +######################## + +FROM ubuntu:22.04 AS build-server + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + ca-certificates \ + tini \ + curl \ + docker.io \ + build-essential \ + gcc \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + # Tini is a small init binary to properly handle signals -CMD ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "start-web-server", "0.0.0.0:80"] +ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] +CMD ["start-build-server"] -################## -# Output stage # -################## +COPY --from=build /build/target/release/cratesfyi /usr/local/bin + +############################ +# Registry watcher stage # +############################ -FROM ubuntu:22.04 AS output +FROM ubuntu:22.04 AS registry-watcher -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - git \ - libmagic1 \ - docker.io \ - ca-certificates \ - build-essential \ - gcc \ - pkg-config \ - libssl-dev +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + ca-certificates \ + tini \ + curl \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Tini is a small init binary to properly handle signals +ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] +CMD ["start-registry-watcher", "--repository-stats-updater=enabled", "--cdn-invalidator=enabled"] + +COPY --from=build /build/target/release/cratesfyi /usr/local/bin + +############### +# CLI stage # +############### + +FROM ubuntu:22.04 AS cli + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + ca-certificates \ + tini \ + && rm -rf /var/lib/apt/lists/* -RUN mkdir -p /opt/docsrs/prefix +ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] COPY --from=build /build/target/release/cratesfyi /usr/local/bin -COPY static /opt/docsrs/static -COPY templates /opt/docsrs/templates -COPY dockerfiles/entrypoint.sh /opt/docsrs/ -COPY vendor /opt/docsrs/vendor - -WORKDIR /opt/docsrs -ENTRYPOINT ["/opt/docsrs/entrypoint.sh"] -CMD ["daemon", "--registry-watcher=disabled"] diff --git a/dockerfiles/entrypoint.sh b/dockerfiles/entrypoint.sh deleted file mode 100755 index b8fdc61aa..000000000 --- a/dockerfiles/entrypoint.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -set -euv - -export DOCSRS_PREFIX=/opt/docsrs/prefix -export DOCSRS_DOCKER=true -export DOCSRS_LOG=${DOCSRS_LOG-"docs-rs,rustwide=info"} -export PATH="$PATH:/build/target/release" - -# Try migrating the database multiple times if it fails -# This avoids the docker container crashing the first time it's started with -# docker-compose, as PostgreSQL needs some time to initialize. -set +e -failed=0 -while true; do - if ! cratesfyi database migrate; then - ((failed=failed + 1)) - if [ "${failed}" -eq 5 ]; then - exit 1 - fi - echo "failed to migrate the database" - echo "waiting 1 second..." - sleep 1 - else - break - fi -done -set -e - -if ! [ -d "${DOCSRS_PREFIX}/crates.io-index/.git" ]; then - git clone ${REGISTRY_URL:-https://github.com/rust-lang/crates.io-index} "${DOCSRS_PREFIX}/crates.io-index" - # Prevent new crates built before the container creation to be built - git --git-dir="$DOCSRS_PREFIX/crates.io-index/.git" branch crates-index-diff_last-seen -fi - -cratesfyi build update-toolchain --only-first-time - -cratesfyi "$@"