CI: lint file generation

Previously it was possible for devs to forget to run `./task generate` which could result in strange CI failures at best or shipping stale manifests in the worst case. This commit adds `git diff --exit-code` to our CI tasks which will cause CI to fail if the generation task produces any unexpected diffs.
redpanda-data · Aug 27, 2024 · 302f988 · 302f988
1 parent 72ba3d3
commit 302f988
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 58 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -47,8 +47,6 @@ steps:
               secret-id: sdlc/prod/buildkite/goreleaser_key
             - json-key: .
               secret-id: sdlc/prod/buildkite/grafana_token
-            - json-key: .
-              secret-id: sdlc/prod/buildkite/quill
             - json-key: .
               secret-id: sdlc/prod/buildkite/redpanda_sample_license
             - json-key: .

diff --git a/ci/docker/nix.Dockerfile b/ci/docker/nix.Dockerfile
@@ -7,7 +7,15 @@ WORKDIR "/work"
 RUN echo 'experimental-features = nix-command flakes' >> /etc/nix/nix.conf \
 	# Fix a weird docker issues. See https://github.com/NixOS/nix/issues/5258
 	&& echo 'filter-syscalls = false' >> /etc/nix/nix.conf \
-	# Fix a weird git/flake issue. See https://github.com/NixOS/nix/issues/10202
-	&& git config --global --add safe.directory .
+	# Git behaves a bit strangely if the directory holding the .git folder is
+	# owned by someone other than the running user.
+	# (https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9)
+	# This affects us in two ways:
+	# 1. A weird interaction with nix flakes (https://github.com/NixOS/nix/issues/10202)
+	# 2. Trying to run git commands from within this image for CI builds.
+	# We could try to specify a subset of directories but given that the issues
+	# only pop up in dockerized buildkite builds, it feels fairly safe and much
+	# easier to disable this check entirely:
+	&& git config --global --add safe.directory '*'
 
 ENTRYPOINT ["nix", "develop", "--impure", "--command"]
diff --git a/taskfiles/ci.yml b/taskfiles/ci.yml
@@ -13,24 +13,6 @@ tasks:
     preconditions:
       - test -n "$GITHUB_API_TOKEN" || test -n "$GITHUB_TOKEN"
 
-  push-image-cache:
-    desc: push builder cache
-    summary: |
-      Pushes from CI environments, but restricted to jobs that correspond to
-      merges into upstream branches. The PULL_REQUEST variable takes the
-      value of the PR number when the build is triggered from a PR, and
-      takes a value of 'false' for merges to branches.
-
-      NOTE: In addition to the conditional execution of this task,
-            buildkite pipelines for the redpanda project are
-            configured in such a way that only a subset of branches
-            from the upstream repo trigger jobs.
-    cmds:
-      - docker tag {{.DOCKER_IMAGE}} {{.DOCKER_IMAGE}}-{{.IMAGE_CACHE_TAG_SUFFIX}}
-      - docker push {{.DOCKER_IMAGE}}-{{.IMAGE_CACHE_TAG_SUFFIX}}
-    status:
-      - test "$CI" != "true" -o "$PULL_REQUEST" != "false"
-
   k8s:
     cmds:
       - task: run-k8s-tests
@@ -51,12 +33,12 @@ tasks:
     cmds:
       - 'echo "~~~ Running operator v2 e2e tests :k8s:"'
       - task: configure-git-private-repo
+      - task: :k8s:run-golangci-lint
+      - task: :k8s:generate
+      - task: assert-no-diffs
       - task: :k8s:run-kuttl-tests
         vars:
           KUTTL_CONFIG_FILE: kuttl-v2-test.yaml
-      - task: chown-buildkite-agent
-        vars:
-          DIR: '{{.SRC_DIR}}/src/go/k8s'
       - cp "{{.SRC_DIR}}/src/go/k8s/kuttl-exit-code" ./k8s-stable-test-exit-code
       # fail explicitly if stable operator tests failed
       - "grep -q '0' ./k8s-stable-test-exit-code"
@@ -68,9 +50,6 @@ tasks:
       - task: :k8s:run-kuttl-tests
         vars:
           KUTTL_CONFIG_FILE: kuttl-v2-helm-test.yaml
-      - task: chown-buildkite-agent
-        vars:
-          DIR: '{{.SRC_DIR}}/src/go/k8s'
       - cp "{{.SRC_DIR}}/src/go/k8s/kuttl-exit-code" ./k8s-stable-test-exit-code
       # fail explicitly if stable operator tests failed
       - "grep -q '0' ./k8s-stable-test-exit-code"
@@ -80,13 +59,12 @@ tasks:
       - 'echo "~~~ Linting operator code :golangci-lint:"'
       - task: configure-git-private-repo
       - task: :k8s:run-golangci-lint
+      - task: :k8s:generate
+      - task: assert-no-diffs
       - 'echo "~~~ Run operator code unit tests :golang:"'
       - task: :k8s:run-unit-tests
       - 'echo "~~~ Running operator e2e tests :k8s:"'
       - task: :k8s:run-kuttl-tests
-      - task: chown-buildkite-agent
-        vars:
-          DIR: '{{.SRC_DIR}}/src/go/k8s'
       - cp "{{.SRC_DIR}}/src/go/k8s/kuttl-exit-code" ./k8s-stable-test-exit-code
       # fail explicitly if stable operator tests failed
       - "grep -q '0' ./k8s-stable-test-exit-code"
@@ -101,10 +79,12 @@ tasks:
     status:
       - test -z '{{.TAG_NAME}}' # only run for tagged commits
 
-  chown-buildkite-agent:
+  assert-no-diffs:
+    desc: "Fail on any unexpected diffs to generated files (CI only)"
     cmds:
-      - chown -R 2000:2000 '{{.DIR}}'
-    preconditions:
-      - test -n '{{.DIR}}'
+    - ls -lah # Debugging
+    - ls -lah .git # Debugging
+    - git status # Debugging
+    - git diff --exit-code
     status:
-      - test "$CI" != "true" # run only in CI
+      - test "$CI" != "true" # run only in CI as local runs may have unstaged changes.
diff --git a/taskfiles/dev.yml b/taskfiles/dev.yml
@@ -18,11 +18,8 @@ vars:
   KUTTL_INSTALL_DIR: '{{.BUILD_ROOT}}/tools/kuttl/{{.KUTTL_VERSION}}'
   GORELEASER_INSTALL_DIR: '{{.BUILD_ROOT}}/tools/goreleaser'
   GORELEASER_VERSION: '1.20.0'
-  QUILL_INSTALL_DIR: '{{.BUILD_ROOT}}/tools/quill'
-  QUILL_VERSION: '0.4.1'
   DOCKER_TAG_LIST_INSTALL_DIR: '{{.BUILD_ROOT}}/tools/docker-tag-list'
 
-
 tasks:
   install-task:
     cmds:
@@ -186,15 +183,3 @@ tasks:
       - |
         PATH={{.GORELEASER_INSTALL_DIR}}:$PATH
         [[ $(goreleaser --version | grep -o {{.GORELEASER_VERSION}}) == {{.GORELEASER_VERSION}} ]]
-
-  install-quill:
-    desc: install quill
-    cmds:
-      - mkdir -p '{{.QUILL_INSTALL_DIR}}'
-      - |
-        curl -sSfL \
-          https://raw.githubusercontent.com/anchore/quill/main/install.sh \
-          | sh -s -- -b "{{.QUILL_INSTALL_DIR}}" "v{{.QUILL_VERSION}}"
-    status:
-      - test -f {{.QUILL_INSTALL_DIR}}/quill
-      - "[[ $({{.QUILL_INSTALL_DIR}}/quill --version | grep -o {{.QUILL_VERSION}}) == {{.QUILL_VERSION}} ]]"
diff --git a/taskfiles/goreleaser.yml b/taskfiles/goreleaser.yml
@@ -6,7 +6,7 @@ tasks:
     desc: release go binaries using the .goreleaser.yml file in given directory
     deps:
       - task: :dev:install-goreleaser
-      - task: :dev:install-quill
+      - task: :k8s:generate
     env:
       GORELEASER_CURRENT_TAG: '{{.TAG_NAME}}'
       GITHUB_TOKEN:
@@ -17,14 +17,9 @@ tasks:
         mkdir -p /tmp
         echo "" >> "$EMPTY_NOTES"
 
-        export PATH="{{.GORELEASER_INSTALL_DIR}}:{{.QUILL_INSTALL_DIR}}:$PATH"
+        export PATH="{{.GORELEASER_INSTALL_DIR}}:$PATH"
         goreleaser release --clean --release-notes "$EMPTY_NOTES"
     preconditions:
-      - test -n "$QUILL_SIGN_P12"
-      - test -n "$QUILL_SIGN_PASSWORD"
-      - test -n "$QUILL_NOTARY_KEY_ID"
-      - test -n "$QUILL_NOTARY_KEY"
-      - test -n "$QUILL_NOTARY_ISSUER"
       - test -n "$GITHUB_API_TOKEN" || test -n "$GITHUB_TOKEN"
       - test -n "{{.TAG_NAME}}"
 
@@ -34,6 +29,7 @@ tasks:
     desc: build go binaries with goreleaser and a given id
     deps:
       - task: :dev:install-goreleaser
+      - task: :k8s:generate
     env:
       GORELEASER_CURRENT_TAG: '{{.TAG_NAME}}'
     cmds: