feat: Use pipedreams to deploy

.github/workflows/validate-pipelines.yml → .github/workflows/validate-pipelines.yaml

@@ -3,7 +3,7 @@ name: Validate Deployment Pipelines
 on:
     pull_request:
     push:
-        branches: [main, test-me-*]
+        branches: [main]
 
 concurrency:
     group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -25,6 +25,7 @@ jobs:
               filters: |
                 gocd:
                   - 'gocd/**'
+                  - '.github/workflows/validate-pipelines.yml'
 
     validate:
         if: needs.files-changed.outputs.gocd == 'true'
@@ -47,6 +48,11 @@ jobs:
                 token_format: 'id_token'
                 id_token_audience: '610575311308-9bsjtgqg4jm01mt058rncpopujgk3627.apps.googleusercontent.com'
                 id_token_include_email: true
+            - uses: getsentry/action-gocd-jsonnet@v1
+              with:
+                jb-install: true
+                jsonnet-dir: gocd/templates
+                generated-dir: gocd/generated-pipelines
             - uses: getsentry/action-validate-gocd-pipelines@v1
               with:
                 configrepo: vroom__main

.gitignore

@@ -14,5 +14,5 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
-# Dependency directories (remove the comment below to include it)
-# vendor/
+/gocd/templates/vendor/
+/gocd/generated-pipelines/

CHANGELOG.md

@@ -13,6 +13,7 @@
 - Fix android issue frame detection. ([#305](https://github.com/getsentry/vroom/pull/305))
 - Fix backward compatibility with Android profiles without timestamp. ([#307](https://github.com/getsentry/vroom/pull/307))
 - Report all GCS errors again. ([#311](https://github.com/getsentry/vroom/pull/311))
+- Use pipedreams to deploy. ([#312](https://github.com/getsentry/vroom/pull/312))
 
 ## 23.8.0
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build run test issuedetection downloader python-stdlib
+.PHONY: build run test issuedetection downloader python-stdlib gocd
 
 build:
 	./scripts/build.sh
@@ -27,3 +27,18 @@ format:
 
 python-stdlib:
 	python scripts/make_python_stdlib.py
+
+gocd:
+	rm -rf ./gocd/generated-pipelines
+	mkdir -p ./gocd/generated-pipelines
+	cd ./gocd/templates && jb install && jb update
+
+  # Format
+	find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnetfmt -i
+  # Lint
+	find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet-lint -J ./gocd/templates/vendor
+	# Build
+	cd ./gocd/templates && find . -type f \( -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet --ext-code output-files=true -J vendor -m ../generated-pipelines
+
+  # Convert JSON to yaml
+	cd ./gocd/generated-pipelines && find . -type f \( -name '*.yaml' \) -print0 | xargs -n 1 -0 yq -p json -o yaml -i

gocd/templates/bash/check-cloudbuild.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+/devinfra/scripts/checks/googlecloud/checkcloudbuild.py \
+	${GO_REVISION_VROOM_REPO} \
+	"internal-sentry" \
+	"us.gcr.io/internal-sentry/vroom"

gocd/templates/bash/check-github.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+/devinfra/scripts/checks/githubactions/checkruns.py \
+	getsentry/vroom \
+	${GO_REVISION_VROOM_REPO} \
+	test-vroom

gocd/templates/bash/deploy.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}")
+
+/devinfra/scripts/k8s/k8stunnel
+/devinfra/scripts/k8s/k8s-deploy.py \
+	--type="cronjob" \
+	--label-selector="${LABEL_SELECTOR}" \
+	--image="us.gcr.io/internal-sentry/vroom:${GO_REVISION_VROOM_REPO}" \
+	--container-name="vroom"

gocd/templates/bash/wait-canary.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+/devinfra/scripts/canary/canarychecks.py \
+  --skip-check=${SKIP_CANARY_CHECKS} \
+  --wait-minutes=5

gocd/templates/jsonnetfile.json

@@ -0,0 +1,15 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/getsentry/gocd-jsonnet.git",
+          "subdir": "libs"
+        }
+      },
+      "version": "v2.2.2"
+    }
+  ],
+  "legacyImports": true
+}

gocd/templates/jsonnetfile.lock.json

@@ -0,0 +1,16 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/getsentry/gocd-jsonnet.git",
+          "subdir": "libs"
+        }
+      },
+      "version": "54bb0adceffb690bf4aa744b23acdb7888c51fd5",
+      "sum": "stYmA7r5/MARC5qkFFR+94R1W5juwjmRNVQFOTxmwVA="
+    }
+  ],
+  "legacyImports": false
+}

gocd/templates/pipelines/vroom.libsonnet

@@ -0,0 +1,71 @@
+local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/libs/gocd-tasks.libsonnet';
+
+function(region) {
+  environment_variables: {
+    GITHUB_TOKEN: '{{SECRET:[devinfra-github][token]}}',
+    SENTRY_REGION: region,
+    SKIP_CANARY_CHECKS: false,
+  },
+  materials: {
+    vroom_repo: {
+      git: '[email protected]:getsentry/vroom.git',
+      shallow_clone: true,
+      branch: 'main',
+      destination: 'vroom',
+    },
+  },
+  lock_behavior: 'unlockWhenFinished',
+  stages: [
+    {
+      checks: {
+        fetch_materials: true,
+        jobs: {
+          deploy: {
+            timeout: 600,
+            elastic_profile_id: 'vroom',
+            tasks: [
+              gocdtasks.script(importstr '../bash/check-github.sh'),
+              gocdtasks.script(importstr '../bash/check-cloudbuild.sh'),
+            ],
+          },
+        },
+      },
+    },
+    {
+      'deploy-canary': {
+        fetch_materials: true,
+        jobs: {
+          deploy: {
+            timeout: 600,
+            elastic_profile_id: 'vroom',
+            environment_variables: {
+              LABEL_SELECTOR: 'service=vroom,environment=production,env=canary',
+              WAIT_MINUTES: '5',
+            },
+            tasks: [
+              gocdtasks.script(importstr '../bash/deploy.sh'),
+              gocdtasks.script(importstr '../bash/wait-canary.sh'),
+            ],
+          },
+        },
+      },
+    },
+    {
+      'deploy-primary': {
+        fetch_materials: true,
+        jobs: {
+          deploy: {
+            timeout: 600,
+            elastic_profile_id: 'vroom',
+            environment_variables: {
+              LABEL_SELECTOR: 'service=vroom,environment=production',
+            },
+            tasks: [
+              gocdtasks.script(importstr '../bash/deploy.sh'),
+            ],
+          },
+        },
+      },
+    },
+  ],
+}

gocd/templates/vroom.jsonnet

@@ -0,0 +1,30 @@
+local vroom = import './pipelines/vroom.libsonnet';
+local pipedream = import 'github.com/getsentry/gocd-jsonnet/libs/pipedream.libsonnet';
+
+local pipedream_config = {
+  name: 'vroom',
+  exclude_regions: [
+    's4s',
+    'customer-1',
+    'customer-2',
+    'customer-3',
+    'customer-4',
+    'customer-5',
+    'customer-6',
+  ],
+  materials: {
+    vroom_repo: {
+      git: '[email protected]:getsentry/vroom.git',
+      shallow_clone: true,
+      branch: 'main',
+      destination: 'vroom',
+    },
+  },
+  rollback: {
+    material_name: 'vroom_repo',
+    stage: 'deploy-primary',
+    elastic_profile_id: 'vroom',
+  },
+};
+
+pipedream.render(pipedream_config, vroom)