diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yaml similarity index 100% rename from .github/workflows/ci.yml rename to .github/workflows/ci.yaml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yaml similarity index 100% rename from .github/workflows/codeql.yml rename to .github/workflows/codeql.yaml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yaml similarity index 100% rename from .github/workflows/release.yml rename to .github/workflows/release.yaml diff --git a/.github/workflows/validate-pipelines.yml b/.github/workflows/validate-pipelines.yaml similarity index 86% rename from .github/workflows/validate-pipelines.yml rename to .github/workflows/validate-pipelines.yaml index e579e90..7269597 100644 --- a/.github/workflows/validate-pipelines.yml +++ b/.github/workflows/validate-pipelines.yaml @@ -3,7 +3,7 @@ name: Validate Deployment Pipelines on: pull_request: push: - branches: [main, test-me-*] + branches: [main] concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -25,6 +25,7 @@ jobs: filters: | gocd: - 'gocd/**' + - '.github/workflows/validate-pipelines.yml' validate: if: needs.files-changed.outputs.gocd == 'true' @@ -47,6 +48,11 @@ jobs: token_format: 'id_token' id_token_audience: '610575311308-9bsjtgqg4jm01mt058rncpopujgk3627.apps.googleusercontent.com' id_token_include_email: true + - uses: getsentry/action-gocd-jsonnet@v1 + with: + jb-install: true + jsonnet-dir: gocd/templates + generated-dir: gocd/generated-pipelines - uses: getsentry/action-validate-gocd-pipelines@v1 with: configrepo: vroom__main diff --git a/.gitignore b/.gitignore index ca0dc1b..eb00c23 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,5 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out -# Dependency directories (remove the comment below to include it) -# vendor/ +/gocd/templates/vendor/ +/gocd/generated-pipelines/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 61103bd..9a237e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Fix android issue frame detection. ([#305](https://github.com/getsentry/vroom/pull/305)) - Fix backward compatibility with Android profiles without timestamp. ([#307](https://github.com/getsentry/vroom/pull/307)) - Report all GCS errors again. ([#311](https://github.com/getsentry/vroom/pull/311)) +- Use pipedreams to deploy. ([#312](https://github.com/getsentry/vroom/pull/312)) ## 23.8.0 diff --git a/Makefile b/Makefile index ac7caf3..b9bb92b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build run test issuedetection downloader python-stdlib +.PHONY: build run test issuedetection downloader python-stdlib gocd build: ./scripts/build.sh @@ -27,3 +27,18 @@ format: python-stdlib: python scripts/make_python_stdlib.py + +gocd: + rm -rf ./gocd/generated-pipelines + mkdir -p ./gocd/generated-pipelines + cd ./gocd/templates && jb install && jb update + + # Format + find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnetfmt -i + # Lint + find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet-lint -J ./gocd/templates/vendor + # Build + cd ./gocd/templates && find . -type f \( -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet --ext-code output-files=true -J vendor -m ../generated-pipelines + + # Convert JSON to yaml + cd ./gocd/generated-pipelines && find . -type f \( -name '*.yaml' \) -print0 | xargs -n 1 -0 yq -p json -o yaml -i diff --git a/gocd/pipelines/cd.yaml b/gocd/pipelines/cd.yaml deleted file mode 100644 index f5b5a95..0000000 --- a/gocd/pipelines/cd.yaml +++ /dev/null @@ -1,78 +0,0 @@ -format_version: 10 -pipelines: - deploy-vroom: - environment_variables: - GITHUB_TOKEN: "{{SECRET:[devinfra-github][token]}}" - GCP_PROJECT: internal-sentry - GKE_CLUSTER: zdpwkxst - GKE_REGION: us-central1 - GKE_CLUSTER_ZONE: b - GKE_BASTION_ZONE: b - ENVIRONMENT: production - group: vroom - lock_behavior: unlockWhenFinished - materials: - vroom_repo: - git: git@github.com:getsentry/vroom.git - shallow_clone: true - destination: vroom - branch: main - stages: - - checks: - fetch_materials: true - jobs: - check_ci_status: - timeout: 600 - elastic_profile_id: vroom - tasks: - - script: | - /devinfra/scripts/checks/githubactions/checkruns.py \ - getsentry/vroom \ - ${GO_REVISION_VROOM_REPO} \ - test-vroom - check_build_status: - timeout: 600 - elastic_profile_id: vroom - tasks: - - script: | - /devinfra/scripts/checks/googlecloud/checkcloudbuild.py \ - ${GO_REVISION_VROOM_REPO} \ - ${GCP_PROJECT} \ - "us.gcr.io/${GCP_PROJECT}/vroom" - - deploy-canary: - approval: - type: success - allow_only_on_success: true - jobs: - deploy-to-k8s: - timeout: 600 - elastic_profile_id: vroom - tasks: - - script: | - /devinfra/scripts/k8s/k8stunnel - - /devinfra/scripts/k8s/k8s-deploy.py \ - --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \ - --label-selector="service=vroom,component=default,environment=${ENVIRONMENT},env=canary" \ - --image="us.gcr.io/${GCP_PROJECT}/vroom:${GO_REVISION_VROOM_REPO}" \ - --container-name="vroom" - - script: | - /devinfra/scripts/canary/canarychecks.py \ - --wait-minutes=5 - - deploy-primary: - approval: - type: success - allow_only_on_success: true - jobs: - deploy-to-k8s: - timeout: 600 - elastic_profile_id: vroom - tasks: - - script: | - /devinfra/scripts/k8s/k8stunnel - - /devinfra/scripts/k8s/k8s-deploy.py \ - --context="gke_${GCP_PROJECT}_${GKE_REGION}-${GKE_CLUSTER_ZONE}_${GKE_CLUSTER}" \ - --label-selector="service=vroom,component=default,environment=${ENVIRONMENT}" \ - --image="us.gcr.io/${GCP_PROJECT}/vroom:${GO_REVISION_VROOM_REPO}" \ - --container-name="vroom" diff --git a/gocd/templates/bash/check-cloudbuild.sh b/gocd/templates/bash/check-cloudbuild.sh new file mode 100644 index 0000000..6538466 --- /dev/null +++ b/gocd/templates/bash/check-cloudbuild.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +/devinfra/scripts/checks/googlecloud/checkcloudbuild.py \ + ${GO_REVISION_VROOM_REPO} \ + "internal-sentry" \ + "us.gcr.io/internal-sentry/vroom" diff --git a/gocd/templates/bash/check-github.sh b/gocd/templates/bash/check-github.sh new file mode 100644 index 0000000..bc2c642 --- /dev/null +++ b/gocd/templates/bash/check-github.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +/devinfra/scripts/checks/githubactions/checkruns.py \ + getsentry/vroom \ + ${GO_REVISION_VROOM_REPO} \ + test-vroom diff --git a/gocd/templates/bash/deploy.sh b/gocd/templates/bash/deploy.sh new file mode 100644 index 0000000..1a09ed5 --- /dev/null +++ b/gocd/templates/bash/deploy.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") + +/devinfra/scripts/k8s/k8stunnel +/devinfra/scripts/k8s/k8s-deploy.py \ + --type="cronjob" \ + --label-selector="${LABEL_SELECTOR}" \ + --image="us.gcr.io/internal-sentry/vroom:${GO_REVISION_VROOM_REPO}" \ + --container-name="vroom" diff --git a/gocd/templates/bash/wait-canary.sh b/gocd/templates/bash/wait-canary.sh new file mode 100644 index 0000000..d7929fb --- /dev/null +++ b/gocd/templates/bash/wait-canary.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +/devinfra/scripts/canary/canarychecks.py \ + --skip-check=${SKIP_CANARY_CHECKS} \ + --wait-minutes=5 diff --git a/gocd/templates/jsonnetfile.json b/gocd/templates/jsonnetfile.json new file mode 100644 index 0000000..736010c --- /dev/null +++ b/gocd/templates/jsonnetfile.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/getsentry/gocd-jsonnet.git", + "subdir": "libs" + } + }, + "version": "v2.2.2" + } + ], + "legacyImports": true +} diff --git a/gocd/templates/jsonnetfile.lock.json b/gocd/templates/jsonnetfile.lock.json new file mode 100644 index 0000000..b477e85 --- /dev/null +++ b/gocd/templates/jsonnetfile.lock.json @@ -0,0 +1,16 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/getsentry/gocd-jsonnet.git", + "subdir": "libs" + } + }, + "version": "54bb0adceffb690bf4aa744b23acdb7888c51fd5", + "sum": "stYmA7r5/MARC5qkFFR+94R1W5juwjmRNVQFOTxmwVA=" + } + ], + "legacyImports": false +} diff --git a/gocd/templates/pipelines/vroom.libsonnet b/gocd/templates/pipelines/vroom.libsonnet new file mode 100644 index 0000000..a2a9042 --- /dev/null +++ b/gocd/templates/pipelines/vroom.libsonnet @@ -0,0 +1,71 @@ +local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/libs/gocd-tasks.libsonnet'; + +function(region) { + environment_variables: { + GITHUB_TOKEN: '{{SECRET:[devinfra-github][token]}}', + SENTRY_REGION: region, + SKIP_CANARY_CHECKS: false, + }, + materials: { + vroom_repo: { + git: 'git@github.com:getsentry/vroom.git', + shallow_clone: true, + branch: 'main', + destination: 'vroom', + }, + }, + lock_behavior: 'unlockWhenFinished', + stages: [ + { + checks: { + fetch_materials: true, + jobs: { + deploy: { + timeout: 600, + elastic_profile_id: 'vroom', + tasks: [ + gocdtasks.script(importstr '../bash/check-github.sh'), + gocdtasks.script(importstr '../bash/check-cloudbuild.sh'), + ], + }, + }, + }, + }, + { + 'deploy-canary': { + fetch_materials: true, + jobs: { + deploy: { + timeout: 600, + elastic_profile_id: 'vroom', + environment_variables: { + LABEL_SELECTOR: 'service=vroom,environment=production,env=canary', + WAIT_MINUTES: '5', + }, + tasks: [ + gocdtasks.script(importstr '../bash/deploy.sh'), + gocdtasks.script(importstr '../bash/wait-canary.sh'), + ], + }, + }, + }, + }, + { + 'deploy-primary': { + fetch_materials: true, + jobs: { + deploy: { + timeout: 600, + elastic_profile_id: 'vroom', + environment_variables: { + LABEL_SELECTOR: 'service=vroom,environment=production', + }, + tasks: [ + gocdtasks.script(importstr '../bash/deploy.sh'), + ], + }, + }, + }, + }, + ], +} diff --git a/gocd/templates/vroom.jsonnet b/gocd/templates/vroom.jsonnet new file mode 100644 index 0000000..92c48e4 --- /dev/null +++ b/gocd/templates/vroom.jsonnet @@ -0,0 +1,30 @@ +local vroom = import './pipelines/vroom.libsonnet'; +local pipedream = import 'github.com/getsentry/gocd-jsonnet/libs/pipedream.libsonnet'; + +local pipedream_config = { + name: 'vroom', + exclude_regions: [ + 's4s', + 'customer-1', + 'customer-2', + 'customer-3', + 'customer-4', + 'customer-5', + 'customer-6', + ], + materials: { + vroom_repo: { + git: 'git@github.com:getsentry/vroom.git', + shallow_clone: true, + branch: 'main', + destination: 'vroom', + }, + }, + rollback: { + material_name: 'vroom_repo', + stage: 'deploy-primary', + elastic_profile_id: 'vroom', + }, +}; + +pipedream.render(pipedream_config, vroom)