From 5760273801d0241e1dbce9e7a6c40f687d727f8f Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Tue, 2 Jul 2024 22:42:27 -0700 Subject: [PATCH 01/17] Added flag for not running hadolint. --- build.sh | 7 ++++++- docker/Dockerfile | 17 ++++++++++------- scripts/verify_codebase.sh | 8 +++++--- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/build.sh b/build.sh index cdd43c04..843ad1cc 100755 --- a/build.sh +++ b/build.sh @@ -175,6 +175,11 @@ while [[ $# -gt 0 ]]; do shift 2 ;; + --no-hado) + NO_HADO="x" + shift 1 + ;; + --help) print_help exit 0 @@ -202,7 +207,7 @@ fi if [[ -n "$VERIFY_CODEBASE" ]]; then BUILD_TASKS+=('./scripts/setup_database.sh') - BUILD_TASKS+=('./scripts/verify_codebase.sh') + BUILD_TASKS+=("./scripts/verify_codebase.sh ${NO_HADO:+--no-hado}") fi BUILD_COMMAND="$(join_array ' && ' "${BUILD_TASKS[@]}")" diff --git a/docker/Dockerfile b/docker/Dockerfile index e4c8eee4..a1e0f90b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,9 +1,12 @@ # 'builder' includes all build & testing dependencies. -FROM centos:7 AS builder - -RUN yum -y --setopt=skip_missing_names_on_install=False install epel-release gcc-4.8.5 &&\ - yum -y --setopt=skip_missing_names_on_install=False install ShellCheck-0.3.8 &&\ - yum -y --setopt=skip_missing_names_on_install=False install git-1.8.3.1 &&\ +FROM --platform=linux/amd64 centos:7 AS builder + +RUN yum -y --setopt=skip_missing_names_on_install=False install \ + epel-release \ + gcc-4.8.5 &&\ + yum -y --setopt=skip_missing_names_on_install=False install \ + ShellCheck-0.3.8 \ + git-1.8.3.1 &&\ yum -y clean all ARG FDB_LIB_URL @@ -36,7 +39,7 @@ RUN curl -Lo /usr/local/bin/jq $JQ_URL &&\ # 'gobuild' executes 'go build'. -FROM builder AS gobuild +FROM --platform=linux/amd64 builder AS gobuild COPY . /src WORKDIR /src @@ -46,7 +49,7 @@ RUN go build -o /fdbq -ldflags="-X 'github.com/janderland/fdbq/internal/app.Vers # The final stage builds the 'fdbq' image. -FROM centos:7 +FROM --platform=linux/amd64 centos:7 ARG FDB_LIB_URL RUN curl -Lo fdb.rpm $FDB_LIB_URL &&\ diff --git a/scripts/verify_codebase.sh b/scripts/verify_codebase.sh index cc9f5353..6b8d2054 100755 --- a/scripts/verify_codebase.sh +++ b/scripts/verify_codebase.sh @@ -9,10 +9,12 @@ set -x # Lint shell scripts. find . -type f -iname '*.sh' -print0 | xargs -t -0 shellcheck -# Lint Dockerfiles. -find . -type f -iname 'Dockerfile' -print0 | xargs -t -0 -n 1 hadolint +# Lint Dockerfiles, if the '--no-hado' flag wasn't passed. +if [[ "${1:-}" != '--no-hado' ]]; then + find . -type f -iname 'Dockerfile' -print0 | xargs -t -0 -n 1 hadolint +fi -# build, lint, & test Go code. +# Build, lint, & test Go code. go build ./... golangci-lint run ./... go test ./... From d3574e66abcaa58dec076c2fb9f9cabf39f45607 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Thu, 4 Jul 2024 13:23:51 -0700 Subject: [PATCH 02/17] Switched over to Debian as the base image. --- .env | 2 +- .github/workflows/verify.yml | 8 ++++---- docker/Dockerfile | 30 +++++++++++++++--------------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.env b/.env index 7271c3f2..4d1f9f55 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ FDB_VER=6.2.30 -FDB_LIB_URL=https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients-6.2.30-1.el7.x86_64.rpm +FDB_LIB_URL=https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb FDB_DOCKER_IMAGE=foundationdb/foundationdb:6.2.30 GO_URL=https://go.dev/dl/go1.19.1.linux-amd64.tar.gz GOLANGCI_LINT_VER=v1.49.0 diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 3145a6a8..a4c1c118 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -11,11 +11,11 @@ jobs: fdb_ver: [6.2.30, 7.1.33] include: - fdb_ver: 6.2.30 - fdb_lib_url: https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients-6.2.30-1.el7.x86_64.rpm + fdb_lib_url: https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb fdb_docker_image: foundationdb/foundationdb:6.2.30 - - fdb_ver: 7.1.33 - fdb_lib_url: https://github.com/apple/foundationdb/releases/download/7.1.33/foundationdb-clients-7.1.33-1.el7.x86_64.rpm - fdb_docker_image: foundationdb/foundationdb:7.1.33 + - fdb_ver: 7.1.61 + fdb_lib_url: https://github.com/apple/foundationdb/releases/download/7.1.61/foundationdb-clients_7.1.61-1_amd64.deb + fdb_docker_image: foundationdb/foundationdb:7.1.61 runs-on: ubuntu-latest steps: - name: Checkout diff --git a/docker/Dockerfile b/docker/Dockerfile index a1e0f90b..f690145c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,18 +1,18 @@ # 'builder' includes all build & testing dependencies. -FROM --platform=linux/amd64 centos:7 AS builder +FROM --platform=linux/amd64 debian:12 AS builder -RUN yum -y --setopt=skip_missing_names_on_install=False install \ - epel-release \ - gcc-4.8.5 &&\ - yum -y --setopt=skip_missing_names_on_install=False install \ - ShellCheck-0.3.8 \ - git-1.8.3.1 &&\ - yum -y clean all +RUN apt update &&\ + apt install -y \ + build-essential \ + shellcheck=0.9.0-1 \ + git=1:2.39.2-1.1 \ + curl=7.88.1-10+deb12u6 &&\ + apt clean ARG FDB_LIB_URL -RUN curl -Lo fdb.rpm $FDB_LIB_URL &&\ - rpm -i ./fdb.rpm &&\ - rm ./fdb.rpm +RUN curl -Lo fdb.deb $FDB_LIB_URL &&\ + dpkg -i ./fdb.deb &&\ + rm ./fdb.deb ARG GO_URL RUN curl -Lo go.tar.gz $GO_URL &&\ @@ -49,12 +49,12 @@ RUN go build -o /fdbq -ldflags="-X 'github.com/janderland/fdbq/internal/app.Vers # The final stage builds the 'fdbq' image. -FROM --platform=linux/amd64 centos:7 +FROM --platform=linux/amd64 debian:12 ARG FDB_LIB_URL -RUN curl -Lo fdb.rpm $FDB_LIB_URL &&\ - rpm -i ./fdb.rpm &&\ - rm ./fdb.rpm +RUN curl -Lo fdb.deb $FDB_LIB_URL &&\ + dpkg -i ./fdb.deb &&\ + rm ./fdb.deb ENV TERM="xterm-256color" From b4208789aa53a404848c31afd4e37572536e8ff1 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Thu, 4 Jul 2024 13:24:04 -0700 Subject: [PATCH 03/17] Disabled shellcheck rule for offending line. --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index 843ad1cc..79fe18e8 100755 --- a/build.sh +++ b/build.sh @@ -88,6 +88,7 @@ function join_array { function escape_quotes { out=() for arg in "$@"; do + # shellcheck disable=SC1003 out+=("$(printf "'%s'" "${arg//'/\\'}")") done echo "${out[@]}" From 6717ec2a48d62bdb07ca4bc91ab57b9c20113496 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Thu, 4 Jul 2024 13:28:39 -0700 Subject: [PATCH 04/17] Updated FDB version list. --- .github/workflows/verify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index a4c1c118..8b3fdbec 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -8,7 +8,7 @@ jobs: build: strategy: matrix: - fdb_ver: [6.2.30, 7.1.33] + fdb_ver: [6.2.30, 7.1.61] include: - fdb_ver: 6.2.30 fdb_lib_url: https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb From 020d508b6d02d62d001b273cb2bf7b30e634d7fd Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Fri, 5 Jul 2024 14:56:49 -0700 Subject: [PATCH 05/17] Removed deprecated field. --- docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 1a5ac738..02661643 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,3 @@ -version: "3.9" services: # The build service is responsible for building, From 2d05344eeda9dd0fa1e10bfcf8316ab22f719f96 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Fri, 5 Jul 2024 19:08:16 -0700 Subject: [PATCH 06/17] Explicitly set platform when running. --- docker-compose.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 02661643..72312fcb 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,6 +5,7 @@ services: build: container_name: "build" image: "docker.io/janderland/fdbq-build:${DOCKER_TAG}" + platform: "linux/amd64" build: context: "./docker" target: "builder" @@ -29,6 +30,7 @@ services: fdbq: container_name: "fdbq" image: "docker.io/janderland/fdbq:${DOCKER_TAG}" + platform: "linux/amd64" build: context: "." dockerfile: "./docker/Dockerfile" @@ -49,5 +51,6 @@ services: fdb: container_name: "fdb" image: "${FDB_DOCKER_IMAGE}" + platform: "linux/amd64" ports: - "4500:4500" From 525deb84a9e27a0200f02a25806bd1fe3cc6f078 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Fri, 5 Jul 2024 19:10:22 -0700 Subject: [PATCH 07/17] Define build platform in compose file. --- docker-compose.yaml | 4 ++++ docker/Dockerfile | 16 +++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 72312fcb..8ce3311f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,6 +9,8 @@ services: build: context: "./docker" target: "builder" + platforms: + - "linux/amd64" args: FDB_LIB_URL: "${FDB_LIB_URL}" GO_URL: "${GO_URL}" @@ -34,6 +36,8 @@ services: build: context: "." dockerfile: "./docker/Dockerfile" + platforms: + - "linux/amd64" args: FDBQ_VER: "${DOCKER_TAG}" FDB_LIB_URL: "${FDB_LIB_URL}" diff --git a/docker/Dockerfile b/docker/Dockerfile index f690145c..ca8e3204 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ # 'builder' includes all build & testing dependencies. -FROM --platform=linux/amd64 debian:12 AS builder +FROM debian:12 AS builder RUN apt update &&\ apt install -y \ @@ -10,9 +10,8 @@ RUN apt update &&\ apt clean ARG FDB_LIB_URL -RUN curl -Lo fdb.deb $FDB_LIB_URL &&\ - dpkg -i ./fdb.deb &&\ - rm ./fdb.deb +RUN curl -Lo /fdb.deb $FDB_LIB_URL &&\ + dpkg -i /fdb.deb ARG GO_URL RUN curl -Lo go.tar.gz $GO_URL &&\ @@ -49,12 +48,11 @@ RUN go build -o /fdbq -ldflags="-X 'github.com/janderland/fdbq/internal/app.Vers # The final stage builds the 'fdbq' image. -FROM --platform=linux/amd64 debian:12 +FROM debian:12 -ARG FDB_LIB_URL -RUN curl -Lo fdb.deb $FDB_LIB_URL &&\ - dpkg -i ./fdb.deb &&\ - rm ./fdb.deb +COPY --from=gobuild /fdb.deb /fdb.deb +RUN dpkg -i ./fdb.deb &&\ + rm /fdb.deb ENV TERM="xterm-256color" From 258a5c2acb751dd0112d862a8e84b1fdcdba5f04 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Sat, 6 Jul 2024 12:08:14 -0700 Subject: [PATCH 08/17] Added debug info to build script. --- build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build.sh b/build.sh index 79fe18e8..8095cb35 100755 --- a/build.sh +++ b/build.sh @@ -198,6 +198,12 @@ while [[ $# -gt 0 ]]; do done +# Print helpful debug info. + +id +ls -ld /fdbq + + # Build variables required by the docker compose command. BUILD_TASKS=() From a9b2fb0e84668cc9a028c419a267a96a64fea9cc Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Sat, 6 Jul 2024 12:17:44 -0700 Subject: [PATCH 09/17] Add more debugging. --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index 8095cb35..ee260120 100755 --- a/build.sh +++ b/build.sh @@ -202,6 +202,7 @@ done id ls -ld /fdbq +ls -ld /fdbq/* # Build variables required by the docker compose command. From 6f8325472f51e0082f15977192777d53b19adaf6 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Sat, 6 Jul 2024 12:18:34 -0700 Subject: [PATCH 10/17] Remove failing debug line. --- build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/build.sh b/build.sh index ee260120..90b2aada 100755 --- a/build.sh +++ b/build.sh @@ -201,7 +201,6 @@ done # Print helpful debug info. id -ls -ld /fdbq ls -ld /fdbq/* From e4d6991a1842d7939841f4581481d9eb942b847f Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Sat, 6 Jul 2024 12:39:01 -0700 Subject: [PATCH 11/17] Debug info change. --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 90b2aada..b1113fae 100755 --- a/build.sh +++ b/build.sh @@ -201,7 +201,7 @@ done # Print helpful debug info. id -ls -ld /fdbq/* +ls -ld / # Build variables required by the docker compose command. From 610e4840b5a762e62b005089b7d8448bbfbeadd0 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Sat, 6 Jul 2024 12:42:21 -0700 Subject: [PATCH 12/17] Move debug to correct location. --- build.sh | 6 ------ scripts/verify_codebase.sh | 3 +++ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/build.sh b/build.sh index b1113fae..79fe18e8 100755 --- a/build.sh +++ b/build.sh @@ -198,12 +198,6 @@ while [[ $# -gt 0 ]]; do done -# Print helpful debug info. - -id -ls -ld / - - # Build variables required by the docker compose command. BUILD_TASKS=() diff --git a/scripts/verify_codebase.sh b/scripts/verify_codebase.sh index 6b8d2054..f6700fd5 100755 --- a/scripts/verify_codebase.sh +++ b/scripts/verify_codebase.sh @@ -6,6 +6,9 @@ cd "${0%/*}/.." set -x +id +ls -ld /fdbq + # Lint shell scripts. find . -type f -iname '*.sh' -print0 | xargs -t -0 shellcheck From c28a720e60dd55a5dc66bdf708bbb47c456cc650 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Sat, 6 Jul 2024 12:52:33 -0700 Subject: [PATCH 13/17] Move debugging to different script. --- scripts/verify_codebase.sh | 3 --- scripts/verify_generation.sh | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/verify_codebase.sh b/scripts/verify_codebase.sh index f6700fd5..6b8d2054 100755 --- a/scripts/verify_codebase.sh +++ b/scripts/verify_codebase.sh @@ -6,9 +6,6 @@ cd "${0%/*}/.." set -x -id -ls -ld /fdbq - # Lint shell scripts. find . -type f -iname '*.sh' -print0 | xargs -t -0 shellcheck diff --git a/scripts/verify_generation.sh b/scripts/verify_generation.sh index e5583ee1..24a9ab37 100755 --- a/scripts/verify_generation.sh +++ b/scripts/verify_generation.sh @@ -4,6 +4,9 @@ set -euo pipefail # Change directory to repo root. cd "${0%/*}/.." +id +ls -ld /fdbq + STATUS="$(git status --short)" if [[ -n "$STATUS" ]]; then echo "ERR! Generated code cannot be verified while there are uncommitted changes." From 760313fe0b25a1dec158c5009e24b1bb69c11a73 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Sat, 6 Jul 2024 12:58:13 -0700 Subject: [PATCH 14/17] Remove debbuging. Add git user workaround. --- docker/Dockerfile | 6 ++++++ scripts/verify_generation.sh | 3 --- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ca8e3204..3a5ec936 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,6 +36,12 @@ ARG JQ_URL RUN curl -Lo /usr/local/bin/jq $JQ_URL &&\ chmod +x /usr/local/bin/jq +# Configure git so it allows any user to run git commands +# on the /fdbq directory. This allows the user which runs +# CI to be different from the user which built the Docker +# image. +RUN git config --global --add safe.directory /fdbq + # 'gobuild' executes 'go build'. FROM --platform=linux/amd64 builder AS gobuild diff --git a/scripts/verify_generation.sh b/scripts/verify_generation.sh index 24a9ab37..e5583ee1 100755 --- a/scripts/verify_generation.sh +++ b/scripts/verify_generation.sh @@ -4,9 +4,6 @@ set -euo pipefail # Change directory to repo root. cd "${0%/*}/.." -id -ls -ld /fdbq - STATUS="$(git status --short)" if [[ -n "$STATUS" ]]; then echo "ERR! Generated code cannot be verified while there are uncommitted changes." From b7256d2d1a339d73b8b92f95c477373dd1327401 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Sat, 6 Jul 2024 13:06:16 -0700 Subject: [PATCH 15/17] Fix Dockerfile linting issues. --- docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3a5ec936..b235cb14 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,13 +1,13 @@ # 'builder' includes all build & testing dependencies. FROM debian:12 AS builder -RUN apt update &&\ - apt install -y \ +RUN apt-get update &&\ + apt-get install -y \ build-essential \ shellcheck=0.9.0-1 \ git=1:2.39.2-1.1 \ curl=7.88.1-10+deb12u6 &&\ - apt clean + apt-get clean ARG FDB_LIB_URL RUN curl -Lo /fdb.deb $FDB_LIB_URL &&\ From 391365edf27f5a8f9d9a570173f46bce5034d90a Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Sat, 6 Jul 2024 13:09:48 -0700 Subject: [PATCH 16/17] Fixed Docker linting errors. --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b235cb14..28dff95d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,7 +3,7 @@ FROM debian:12 AS builder RUN apt-get update &&\ apt-get install -y \ - build-essential \ + build-essential=12.9 \ shellcheck=0.9.0-1 \ git=1:2.39.2-1.1 \ curl=7.88.1-10+deb12u6 &&\ @@ -44,7 +44,7 @@ RUN git config --global --add safe.directory /fdbq # 'gobuild' executes 'go build'. -FROM --platform=linux/amd64 builder AS gobuild +FROM builder AS gobuild COPY . /src WORKDIR /src From 48af277db48783d21e6f9d3e3e308488153deff2 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Sat, 6 Jul 2024 13:44:28 -0700 Subject: [PATCH 17/17] Fix more Docker linting errors. --- docker/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 28dff95d..560cd761 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,12 +2,14 @@ FROM debian:12 AS builder RUN apt-get update &&\ - apt-get install -y \ + apt-get install --no-install-recommends -y \ build-essential=12.9 \ + ca-certificates=20230311 \ shellcheck=0.9.0-1 \ git=1:2.39.2-1.1 \ curl=7.88.1-10+deb12u6 &&\ - apt-get clean + apt-get clean &&\ + rm -rf /var/lib/apt/lists/* ARG FDB_LIB_URL RUN curl -Lo /fdb.deb $FDB_LIB_URL &&\