Initial implementation of circle job.

.circleci/config.yml

@@ -0,0 +1,69 @@
+version: 2.1
+
+references:
+  circleci-docker-primary: &circleci-docker-primary trussworks/circleci-docker-primary:385c300cc218b910b6fd2da8041fe848001cecf4
+  postgres: &postgres circleci/postgres:12.3-ram
+
+jobs:
+  validate:
+    docker:
+      - image: *circleci-docker-primary
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - pre-commit-dot-cache-v1-{{ checksum ".pre-commit-config.yaml" }}
+      - run:
+          name: Run pre-commit tests
+          command: pre-commit run --all-files
+      - save_cache:
+          key: pre-commit-dot-cache-v1-{{ checksum ".pre-commit-config.yaml" }}
+          paths:
+            - ~/.cache/pre-commit
+  test:
+    docker:
+      - image: *circleci-docker-primary
+        environment:
+        - PGHOST: 127.0.0.1
+        - PGDATABASE: bork-local
+        - PGPORT: 5432
+        - PGUSER: postgres
+        - PGPASS: postgres
+        - APP_ENV: test
+        - API_PROTOCOL: http
+        - API_HOST: localhost
+        - API_PORT: 8080
+        - PGSSLMODE: disable
+      - image: *postgres
+        environment:
+        - POSTGRES_DB: bork-local
+        - POSTGRES_PASSWORD: postgres
+
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - go-mod-sources-v1-{{ checksum "go.sum" }}
+      - run: wget -qO- https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/6.5.3/flyway-commandline-6.5.3-linux-x64.tar.gz | tar xvz && sudo ln -s `pwd`/flyway-6.5.3/flyway /usr/local/bin 
+      - run:
+          name: Waiting for Postgres to be ready
+          command: |
+            for i in `seq 1 10`;
+            do
+              nc -z localhost 5432 && echo Success && exit 0
+              echo -n .
+              sleep 1
+            done
+            echo Failed waiting for Postgres && exit 1
+      - run: flyway migrate
+      - run: go test ./pkg/...
+      - save_cache:
+          key: go-mod-sources-v1-{{ checksum "go.sum" }}
+          paths:
+            - /go/pkg/mod
+workflows:
+  version: 2.1
+  test:
+    jobs:
+      - validate
+      - test

scripts/build_app

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+#
+# build the `sample` app
+#
+
+builddir="$(git rev-parse --show-toplevel)"
+
+( set -x -u ; go build -a -o "$builddir"/bin/bork "$builddir"/cmd/bork )

scripts/build_db_image

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# for periodically rebuilding the db images
+#
+
+dockerfile="$1"
+repo_name="$2"
+ecr_backend="${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/$repo_name"
+tag="${ecr_backend}:${3:-${CIRCLE_SHA1}}"
+
+builddir="$(git rev-parse --show-toplevel)"
+
+# log in to ECR
+if (set +x -o nounset; aws ecr get-login-password --region "${AWS_DEFAULT_REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${AWS_DEFAULT_REGION}".amazonaws.com) ; then
+  if (set -x ; docker build --quiet --no-cache --tag "$repo_name" "$builddir" --file "$dockerfile") ; then
+    ( set -x ; docker tag "$repo_name" "$tag" && docker push "$tag" )
+  else
+    exit
+  fi
+else
+  exit
+fi

scripts/build_docker_image

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# build `sample` in docker and push to ECR
+#
+
+builddir="$(git rev-parse --show-toplevel)"
+
+ecr_backend="${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/sample-backend"
+
+APPLICATION_VERSION="${CIRCLE_SHA1:-"$(git rev-parse HEAD)"}"
+APPLICATION_DATETIME="$(date --rfc-3339='seconds' --utc)"
+APPLICATION_TS="$(date --date="$APPLICATION_DATETIME" '+%s')"
+
+# log in to ECR
+if (set +x -o nounset; aws ecr get-login-password --region "${AWS_DEFAULT_REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${AWS_DEFAULT_REGION}".amazonaws.com) ; then
+  # build & tag the app image, then push to ECR
+  if (set -x ; docker build --quiet --build-arg ARG_APPLICATION_VERSION="$APPLICATION_VERSION" --build-arg ARG_APPLICATION_DATETIME="$APPLICATION_DATETIME" --build-arg ARG_APPLICATION_TS="$APPLICATION_TS" --no-cache --tag "sample" "$builddir") ; then
+    tag="${ecr_backend}:${APPLICATION_VERSION}"
+    ( set -x ; docker tag "sample" "$tag" && docker push "$tag" )
+  else
+    exit
+  fi
+else
+  exit
+fi

scripts/check_ecr_findings

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+payload="$(jq --null-input ".repository= \"$1\" | .imageTag = \"$2\"")"
+aws lambda invoke --function-name ecr-scan-findings --cli-binary-format raw-in-base64-out --payload "$payload" scan_response.json
+jq < scan_response.json
+totalFindings="$(jq "fromjson | .totalFindings" scan_response.json)"
+if [[ "$totalFindings" != 0 ]]; then
+  echo "Scan found $totalFindings findings!"
+  exit 1
+else
+  echo "Scan found no findings"
+fi

scripts/circleci-announce-broken-branch

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+#####
+## Only alert on master branch
+#####
+[[ $CIRCLE_BRANCH = master ]] || exit 0
+
+NOW=$(date '+%s')
+
+pretext="CircleCI $CIRCLE_BRANCH branch failure!"
+title="CircleCI build #$CIRCLE_BUILD_NUM failed on job $CIRCLE_JOB"
+message="The $CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME $CIRCLE_BRANCH branch broke on job $CIRCLE_JOB! Contact $CIRCLE_USERNAME for more information."
+
+#####
+## Announce in Slack channel
+#####
+# 'color' can be any hex code or the key words 'good', 'warning', or 'danger'
+color="warning"
+if [[ $CIRCLE_JOB = *"deploy"* ]]; then
+  color="danger"
+fi
+
+slack_payload=$(
+cat <<EOM
+{
+    "channel": "#oit-sample-alerts",
+    "attachments": [
+        {
+            "fallback": "$message $CIRCLE_BUILD_URL",
+            "color": "$color",
+            "pretext": "$pretext",
+            "author_name": "$CIRCLE_USERNAME",
+            "title": "$title",
+            "title_link": "$CIRCLE_BUILD_URL",
+            "text": "$message",
+            "ts": $NOW
+        }
+    ]
+}
+EOM
+)
+
+echo
+echo "Slack Payload:"
+echo "$slack_payload"
+echo
+
+curl -X POST --data-urlencode payload="$slack_payload" "$SLACK_WEBHOOK_URL"

scripts/db_lambda_invoke

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+#
+
+set -u
+
+function_name="$1"
+lambda_version="$2"
+service_id="$3"
+
+cluster_id="$APP_ENV-sample-app"
+payload="$(jq --null-input ".command = \"runtask\" | .body.cluster_id = \"$cluster_id\" | .body.service_id = \"$service_id\"")"
+cli_read_timeout=240
+
+if (set -x ; aws lambda invoke --qualifier "$lambda_version" --cli-read-timeout "$cli_read_timeout" --function "$function_name" --cli-binary-format raw-in-base64-out --payload "$payload" lambda_response.json) ; then
+  jq < lambda_response.json
+  exitCode="$(jq '.data.response.taskStatus.exitCode' < lambda_response.json)"
+  if [[ "$exitCode" != 0 ]] ; then
+    echo ": task exited non-zero or exit status could not be parsed" 1>&2
+    exit 1
+  fi
+else
+  exit
+fi

scripts/deploy_service

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# deploy containers to our ecs services tagged with the git digest of the current build
+
+set -u
+
+service_id="$1"
+function_qualifier="$2"
+image="${3:-}"
+
+function_name="$APP_ENV-ecs-manager"
+ecr_backend="${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com"
+
+if [[ -n "$image" ]]; then
+  image_tag="$ecr_backend/$image:${CIRCLE_SHA1-$(git rev-parse HEAD)}"
+fi
+
+_payload() {
+  local image_tag="${1:-}"
+  local cluster_id="${3:-$APP_ENV-sample-app}"
+
+
+  result=$(jq --null-input '.command = "deploy" | .body.cluster_id = "'"$cluster_id"'" | .body.service_ids = ["'"$service_id"'"]') || return
+  if [[ "$service_id" == "sample-app" ]] ; then
+    secrets_regex="^/$APP_ENV/sample-app/\\\S+$"
+    result="$(jq '.body.secrets = ["'"$secrets_regex"'"]' <<< "$result")"
+  fi
+
+  if [[ -n "$image_tag" ]]; then
+    result=$(jq '.body.image = "'"$image_tag"'"' <<< "$result") || return
+  fi
+
+  echo "$result"
+}
+
+_payload "${image_tag:-}" > payload.json || exit
+
+jq < payload.json
+
+echo ": Invoking the $function_name lambda..."
+time (set -x ; aws lambda invoke --function-name "$function_name" --qualifier "$function_qualifier" --cli-binary-format raw-in-base64-out --payload file://./payload.json output.json) || exit
+
+jq < output.json
+
+status="$(jq '.data.response.ResponseMetadata.HTTPStatusCode' < output.json)"
+if [[ "$status" != 200 ]]; then
+  echo ": FATAL HTTPStatusCode is not 200"
+  exit 1
+fi

scripts/healthcheck

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# wait some time for the sample health check http endpoint to return the expected version
+
+if [[ "$APP_ENV" != "prod" ]] ; then
+  url_prefix="${APP_ENV}."
+fi
+
+healthcheck_url="https://${url_prefix}domain.example/api/v1/healthcheck"
+healthcheck_timeout="300"
+
+# wait for the new service to come up
+echo -n "Waiting for health check version to match new deployment on ${healthcheck_url}" 1>&2
+until [[ "$(jq --raw-output --monochrome-output .version < <(curl --silent --show-error --max-time 2 --location "$healthcheck_url"))" == "$CIRCLE_SHA1" ]] ; do
+  if ((++time > healthcheck_timeout)) ; then
+    # we timed out
+    echo ""
+    echo "FATAL: Timed out waiting." 1>&2
+    exit 1
+  else
+    [[ $((time % 10)) -eq 0 ]] && echo -n "."
+    sleep 1
+  fi
+done
+echo ""

scripts/pre-commit-eslint

@@ -0,0 +1,5 @@
+#! /usr/bin/env bash
+
+git diff --cached --name-only --diff-filter=d | \
+  grep -E '\.tsx?$' | \
+  xargs yarn run eslint --ext .ts,.tsx -c .eslintrc --fix

scripts/rds-snapshot-app-db

@@ -0,0 +1,34 @@
+#! /usr/bin/env bash
+#
+#   Creates a snapshot of the app database for the given environment.
+#
+set -eu -o pipefail
+
+readonly environment="$APP_ENV"
+
+readonly db_instance_identifier=sample$environment
+readonly db_snapshot_identifier=$db_instance_identifier-$(date +%s)
+readonly tags=("Key=Environment,Value=$environment" "Key=Tool,Value=$(basename "$0")")
+
+echo
+echo "Wait for concurrent database snapshots for ${db_instance_identifier} to complete before continuing ..."
+time aws rds wait db-snapshot-completed --db-instance-identifier "$db_instance_identifier"
+
+echo
+echo "Create database snapshot for ${db_instance_identifier} with identifier ${db_snapshot_identifier}"
+aws rds create-db-snapshot --db-instance-identifier "$db_instance_identifier" --db-snapshot-identifier "$db_snapshot_identifier" --tags "${tags[@]}"
+
+echo
+echo "Wait for current database snapshot ${db_snapshot_identifier} to complete before continuing ..."
+time aws rds wait db-snapshot-completed --db-snapshot-identifier "$db_snapshot_identifier"
+
+echo
+echo "Describe the database snapshot ${db_snapshot_identifier}"
+db_description=$(aws rds describe-db-snapshots --db-snapshot-identifier "$db_snapshot_identifier")
+echo "${db_description}"
+db_status=$(echo "${db_description}" | jq -r ".DBSnapshots[].Status")
+if [[ "${db_status}" != "available" ]]; then
+  echo
+  echo "DB Status is '${db_status}', expected 'available'"
+  exit 1
+fi

scripts/release_static

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+
+set -eu -o pipefail
+
+case "$APP_ENV" in
+  "dev")
+    STATIC_S3_BUCKET="$STATIC_S3_BUCKET_DEV"
+    ;;
+  "impl")
+    STATIC_S3_BUCKET="$STATIC_S3_BUCKET_IMPL"
+    ;;
+  "prod")
+    STATIC_S3_BUCKET="$STATIC_S3_BUCKET_PROD"
+    ;;
+  *)
+    echo "APP_ENV value not recognized: ${APP_ENV:-unset}"
+    echo "Allowed values: 'dev', 'impl', 'prod'"
+    exit 1
+    ;;
+esac
+
+export REACT_APP_APP_ENV="$APP_ENV"
+
+# Check if we have any access to the s3 bucket
+# Since `s3 ls` returns zero even if the command failed, we assume failure if this command prints anything to stderr
+s3_err="$(aws s3 ls "$STATIC_S3_BUCKET" 1>/dev/null 2>&1)"
+if [[ -z "$s3_err" ]] ; then
+  ( set -x -u ;
+    yarn install
+    yarn run build || exit
+    aws s3 sync --no-progress --delete build/ s3://"$STATIC_S3_BUCKET"/
+  )
+else
+  echo "+ aws s3 ls $STATIC_S3_BUCKET"
+  echo "$s3_err"
+  echo "--"
+  echo "Error reading the S3 bucket. Are you authenticated?" 1>&2
+  exit 1
+fi