From 08c7839dc1c2abf0f0afd7aa74464a452b73c5bd Mon Sep 17 00:00:00 2001 From: Tiffany Lin Date: Tue, 2 Apr 2024 15:26:23 +0000 Subject: [PATCH 1/2] Migrate CI from AWS queues to Google Kubernetes Engine queues - Add necessary k8s plugin config to the pipeline for use with GKE + Buildkite's agent-stack-k8s helm stack - Remove docker agent tag filter - Install bk agent in Dockerfile for use in the code coverage step --- .buildkite/pipeline.yml | 127 ++++++++++++++++++++-------- docker/buildkite/Dockerfile | 15 ++++ docker/buildkite/docker-compose.yml | 1 - 3 files changed, 109 insertions(+), 34 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index eef8f9cc6..9a32f154d 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,81 +1,137 @@ +container: + kubernetes: &kubernetes + gitEnvFrom: + - secretRef: + name: oss-github-ssh-credentials + sidecars: + - image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-dind:v1 + volumeMounts: + - mountPath: /var/run/ + name: docker-sock + securityContext: + privileged: true + allowPrivilegeEscalation: true + mirrorVolumeMounts: true # CRITICAL: this must be at the same indentation level as sidecars + podSpec: &podSpec + containers: + - &commandContainer + image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-command-container:v2 + command: + - |- + echo "Command step was not overridden." + exit 1 + volumeMounts: + - mountPath: /var/run/ + name: docker-sock + resources: + requests: + cpu: 7500m + memory: 28G + volumes: + - name: docker-sock + emptyDir: {} + +agents: + queue: "buildkite-gcp" steps: - label: "fossa analyze" - agents: - queue: "init" - docker: "*" command: ".buildkite/scripts/fossa.sh" - label: ":golang: unit-test" - agents: - queue: "workers" - docker: "*" - commands: - - "make unit_test" - - ".buildkite/scripts/gen_coverage_metadata.sh .build/metadata.txt" artifact_paths: - ".build/*/coverage/*.out" - ".build/cover.out" - ".build/metadata.txt" plugins: + - kubernetes: + <<: *kubernetes + podSpec: + <<: *podSpec + containers: + - <<: *commandContainer + command: + - |- + make unit_test + .buildkite/scripts/gen_coverage_metadata.sh .build/metadata.txt - docker-compose#v3.0.0: run: unit-test config: docker/buildkite/docker-compose.yml - label: ":golangci-lint: validate code is clean" - agents: - queue: "workers" - docker: "*" - command: "./scripts/golint.sh" artifact_paths: [ ] retry: automatic: - limit: 1 + limit: 2 plugins: + - kubernetes: + <<: *kubernetes + podSpec: + <<: *podSpec + containers: + - <<: *commandContainer + command: + - |- + ./scripts/golint.sh - docker-compose#v3.0.0: run: unit-test config: docker/buildkite/docker-compose.yml - label: ":golang: integration-test-sticky-off" - agents: - queue: "workers" - docker: "*" - command: "make integ_test_sticky_off" artifact_paths: - ".build/*/coverage/*.out" retry: automatic: - limit: 1 + limit: 2 plugins: + - kubernetes: + <<: *kubernetes + podSpec: + <<: *podSpec + containers: + - <<: *commandContainer + command: + - |- + make integ_test_sticky_off - docker-compose#v3.0.0: run: integ-test config: docker/buildkite/docker-compose.yml - label: ":golang: integration-test-sticky-on" - agents: - queue: "workers" - docker: "*" - command: "make integ_test_sticky_on" artifact_paths: - ".build/*/coverage/*.out" retry: automatic: - limit: 1 + limit: 2 plugins: + - kubernetes: + <<: *kubernetes + podSpec: + <<: *podSpec + containers: + - <<: *commandContainer + command: + - |- + make integ_test_sticky_on - docker-compose#v3.0.0: run: integ-test config: docker/buildkite/docker-compose.yml - label: ":golang: integration-test-grpc-adapter" - agents: - queue: "workers" - docker: "*" - command: "make integ_test_grpc" artifact_paths: - ".build/*/coverage/*.out" retry: automatic: - limit: 1 + limit: 2 plugins: + - kubernetes: + <<: *kubernetes + podSpec: + <<: *podSpec + containers: + - <<: *commandContainer + command: + - |- + make integ_test_grpc - docker-compose#v3.0.0: run: integ-test-grpc config: docker/buildkite/docker-compose.yml @@ -83,11 +139,16 @@ steps: - wait - label: ":golang: code-coverage" - agents: - queue: "workers" - docker: "*" - command: ".buildkite/scripts/gocov.sh" plugins: + - kubernetes: + <<: *kubernetes + podSpec: + <<: *podSpec + containers: + - <<: *commandContainer + command: + - |- + .buildkite/scripts/gocov.sh - docker-compose#v3.0.0: run: coverage-report config: docker/buildkite/docker-compose.yml diff --git a/docker/buildkite/Dockerfile b/docker/buildkite/Dockerfile index 3968d144f..78c99b099 100644 --- a/docker/buildkite/Dockerfile +++ b/docker/buildkite/Dockerfile @@ -8,3 +8,18 @@ ADD go.mod go.sum /go/src/go.uber.org/cadence/ # allow git-status and similar to work RUN git config --global --add safe.directory /go/src/go.uber.org/cadence RUN GO111MODULE=on go mod download + +# Install dependencies needed to install buildkite-agent +RUN apt-get install -y apt-transport-https dirmngr curl + +# Add Buildkite's GPG key +RUN curl -fsSL https://keys.openpgp.org/vks/v1/by-fingerprint/32A37959C2FA5C3C99EFBC32A79206696452D198 | \ + gpg --dearmor -o /usr/share/keyrings/buildkite-agent-archive-keyring.gpg + +# Add the repository to Apt sources +RUN echo \ + "deb [signed-by=/usr/share/keyrings/buildkite-agent-archive-keyring.gpg] https://apt.buildkite.com/buildkite-agent stable main" | \ + tee /etc/apt/sources.list.d/buildkite-agent.list + +# Install Buildkite agent +RUN apt-get update && apt-get install -yy --no-install-recommends buildkite-agent diff --git a/docker/buildkite/docker-compose.yml b/docker/buildkite/docker-compose.yml index ed1cc414b..38712d97f 100644 --- a/docker/buildkite/docker-compose.yml +++ b/docker/buildkite/docker-compose.yml @@ -116,7 +116,6 @@ services: - "GO111MODULE=on" volumes: - ../../:/go/src/go.uber.org/cadence - - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent networks: services-network: From 33dd4b5ab92b21b5c130f6f9d2382c8bcdb962a4 Mon Sep 17 00:00:00 2001 From: Tiffany Lin Date: Thu, 18 Apr 2024 17:54:24 +0000 Subject: [PATCH 2/2] Add comment on buildkite-agent installation, yq explode usage --- .buildkite/pipeline.yml | 1 + docker/buildkite/Dockerfile | 9 ++------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 9a32f154d..9f86280bd 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,3 +1,4 @@ +# `yq 'explode(.)' .buildkite/pipeline.yml` to view expanded anchors/aliases container: kubernetes: &kubernetes gitEnvFrom: diff --git a/docker/buildkite/Dockerfile b/docker/buildkite/Dockerfile index 78c99b099..86570f243 100644 --- a/docker/buildkite/Dockerfile +++ b/docker/buildkite/Dockerfile @@ -9,17 +9,12 @@ ADD go.mod go.sum /go/src/go.uber.org/cadence/ RUN git config --global --add safe.directory /go/src/go.uber.org/cadence RUN GO111MODULE=on go mod download -# Install dependencies needed to install buildkite-agent +# Install Buildkite agent +# https://buildkite.com/docs/agent/v3/ubuntu RUN apt-get install -y apt-transport-https dirmngr curl - -# Add Buildkite's GPG key RUN curl -fsSL https://keys.openpgp.org/vks/v1/by-fingerprint/32A37959C2FA5C3C99EFBC32A79206696452D198 | \ gpg --dearmor -o /usr/share/keyrings/buildkite-agent-archive-keyring.gpg - -# Add the repository to Apt sources RUN echo \ "deb [signed-by=/usr/share/keyrings/buildkite-agent-archive-keyring.gpg] https://apt.buildkite.com/buildkite-agent stable main" | \ tee /etc/apt/sources.list.d/buildkite-agent.list - -# Install Buildkite agent RUN apt-get update && apt-get install -yy --no-install-recommends buildkite-agent