From c1cfd0333a465c3fcb71d745f843e840fbf96dd7 Mon Sep 17 00:00:00 2001 From: Cameron Smart Date: Mon, 14 Nov 2022 12:04:12 -0800 Subject: [PATCH 1/4] split config file with continuation orb --- .circleci/base_config.yml | 29 ++ .circleci/commands.yml | 355 ++++++++++++++++ .circleci/config.yml | 795 ++--------------------------------- .circleci/generate_config.sh | 8 + .circleci/jobs.yml | 282 +++++++++++++ .circleci/workflows.yml | 121 ++++++ 6 files changed, 819 insertions(+), 771 deletions(-) create mode 100644 .circleci/base_config.yml create mode 100644 .circleci/commands.yml create mode 100644 .circleci/generate_config.sh create mode 100644 .circleci/jobs.yml create mode 100644 .circleci/workflows.yml diff --git a/.circleci/base_config.yml b/.circleci/base_config.yml new file mode 100644 index 000000000..083f8cc87 --- /dev/null +++ b/.circleci/base_config.yml @@ -0,0 +1,29 @@ +version: 2.1 + +orbs: + node: circleci/node@4.7.0 + terraform: circleci/terraform@2.1.0 + jq: circleci/jq@2.2.0 + +executors: + docker-executor: + docker: + - image: cimg/python:3.10.4 + user: root + machine-executor: + machine: + docker_layer_caching: false + image: ubuntu-2204:2022.10.1 + +parameters: + run_dev_deployment: + type: boolean + default: false + + run_owasp_scan: + type: boolean + default: false + + target_env: + type: string + default: '' diff --git a/.circleci/commands.yml b/.circleci/commands.yml new file mode 100644 index 000000000..f7ab324e1 --- /dev/null +++ b/.circleci/commands.yml @@ -0,0 +1,355 @@ +commands: + ### + # General re-usable commands + # + cf-check: + steps: + - run: + name: Ensure cf cli is installed, otherwise install it. + command: sudo ./scripts/cf-check.sh + + docker-compose-check: + steps: + - run: + name: Ensure docker-compose exists, otherwise install it. + command: ./scripts/docker-compose-check.sh + + docker-compose-up-backend: + steps: + - run: + name: Build and spin-up Django API service + command: cd tdrs-backend; docker-compose up -d --build + + docker-compose-up-frontend: + steps: + - run: + name: Build and spin-up React application + command: cd tdrs-frontend; docker-compose up -d --build + + disable-npm-audit: + steps: + - run: + name: Disable npm audit warnings in CI + command: npm set audit false + + # This allows us to use the orb stanza for node/install within other commands + # NOTE: This doesn't work correctly on machine executors + install-nodejs: node/install + + # This allows us to use the node orb to install packages within other commands + install-nodejs-packages: node/install-packages + + install-nodejs-machine: + description: | + Installs our target version of Node.JS using NVM (Node Version Manager) + from the install location provided by machine executor images. + steps: + - run: + name: Install Node.JS + command: | + sudo apt-get update + sudo apt-get install -y libgbm-dev + source /opt/circleci/.nvm/nvm.sh + nvm install v16.13 + nvm alias default v16.13 + echo 'export NVM_DIR="/opt/circleci/.nvm"' >> $BASH_ENV + echo "[ -s \"$NVM_DIR/nvm.sh\" ] && . \"$NVM_DIR/nvm.sh\"" >> $BASH_ENV + + run-owasp-scan: + description: Runs OWASP ZAP scanner and stores resulting artifacts. + parameters: + environment: + description: The environment this script is being run for. + type: enum + enum: ["circle", "nightly"] + target: + description: The target application to be scanned. + type: enum + enum: ["backend", "frontend"] + target_env: + description: The target environment to be scanned. + type: enum + default: "develop" + enum: ["staging", "develop", "prod"] + steps: + - run: + name: Execute OWASP ZAP vulnerability scan + #wait up to 60min for scan + no_output_timeout: 60m + command: ./scripts/zap-scanner.sh <> <> <> + - store_artifacts: + path: tdrs-<>/reports/owasp_report.html + + sudo-check: + steps: + - run: + name: Ensure sudo is installed, otherwise install it. + command: ./scripts/sudo-check.sh + + upload-codecov: + description: Uploads testing code coverage results to Codecov + parameters: + component: + description: The component of the application being tested, either backend or frontend. + type: enum + enum: [ "backend", "frontend" ] + coverage-report: + description: The path to the coverage report being uploaded. + type: string + steps: + - run: + name: Ensure Codecov uploader is installed, otherwise install it. + command: ./scripts/codecov-check.sh + - run: + name: Determine Codecov metric flag + command: | + if [ "$CIRCLE_BRANCH" == "main" ] ; then + CURRENT_FLAG=main-<> + elif [ "$CIRCLE_BRANCH" == "master" ] ; then + CURRENT_FLAG=master-<> + else + CURRENT_FLAG=dev-<> + fi + echo "export CURRENT_FLAG=$CURRENT_FLAG" >> $BASH_ENV + - run: + name: Upload code coverage report if target branch + command: codecov -t "$CODECOV_TOKEN" -f <> -F "$CURRENT_FLAG" + + ### + # Deployment commands + # + login-cloud-dot-gov: + description: Authenticates with Cloud.gov and sets org and space targets + parameters: + cf-password: + type: env_var_name + default: CF_PASSWORD_DEV + cf-org: + type: env_var_name + default: CF_ORG + cf-space: + type: string + default: tanf-dev + cf-username: + type: env_var_name + default: CF_USERNAME_DEV + steps: + - run: + name: Login to Cloud.gov and set application targets + command: | + cf login -a https://api.fr.cloud.gov \ + -u ${<>} \ + -p ${<>} \ + -o ${<>} \ + -s <> + get-app-deploy-strategy: + parameters: + appname: + type: string + steps: + - run: + name: Determine deploy strategy + command: | + # NOTE: The || true is a no-op included to suppress exit codes which + # would cause the step to exit early due to use of pipefail + APP_GUID=$(cf app <> --guid || true) + if [ "$APP_GUID" == "FAILED" ]; then + echo "export DEPLOY_STRATEGY=initial" >> $BASH_ENV + else + echo "export DEPLOY_STRATEGY=rolling" >> $BASH_ENV + fi + deploy-backend: + parameters: + backend-appname: + default: tdp-backend + type: string + cf-space: + default: tanf-dev + type: string + steps: + - get-app-deploy-strategy: + appname: <> + - run: + name: Deploy backend application + command: | + bash ./scripts/deploy-backend.sh \ + $DEPLOY_STRATEGY \ + <> \ + <> + deploy-clamav: + parameters: + backend-appname: + default: tdp-backend + type: string + cf-org: + default: CF_ORG + type: env_var_name + cf-space: + default: tanf-dev + type: string + steps: + - run: + name: Deploy ClamAV REST application + command: | + cf push clamav-rest -f tdrs-backend/manifest.clamav.yml \ + --var cf-space=<> \ + - run: + name: Enable internal route between backend and clamav-rest app + command: | + cf add-network-policy <> clamav-rest \ + -s <> \ + -o ${<>} \ + --protocol tcp \ + --port 9000 + deploy-frontend: + parameters: + environment: + description: The environment to deploy to. + type: enum + enum: [ "development", "production" ] + default: development + backend-appname: + default: tdp-backend + type: string + frontend-appname: + default: tdp-frontend + type: string + +# So the frontend knows what space its in for the banner. +# I am unclear if the domain is a reliable metric to make this function +# It seems like it might not be working + cf-space: + default: dev + type: string + steps: + - install-nodejs: + node-version: "16.13" + - disable-npm-audit + - install-nodejs-packages: + app-dir: tdrs-frontend + - get-app-deploy-strategy: + appname: <> + - run: + name: Deploy frontend application + command: | + bash ./scripts/deploy-frontend.sh \ + $DEPLOY_STRATEGY \ + <> \ + <> \ + <> \ + <> + deploy-cloud-dot-gov: + parameters: + environment: + description: The environment to deploy to. + type: enum + enum: [ "development", "production" ] + default: development + backend-appname: + default: tdp-backend + type: string + cf-password: + default: CF_PASSWORD_DEV + type: env_var_name + cf-org: + default: CF_ORG + type: env_var_name + cf-space: + default: tanf-dev + type: string + cf-username: + default: CF_USERNAME_DEV + type: env_var_name + frontend-appname: + default: tdp-frontend + type: string + steps: + - checkout + - sudo-check + - cf-check + - login-cloud-dot-gov: + cf-password: <> + cf-org: <> + cf-space: <> + cf-username: <> + - deploy-backend: + backend-appname: <> + cf-space: <> + - deploy-clamav: + backend-appname: <> + cf-org: <> + cf-space: <> + - deploy-frontend: + environment: <> + backend-appname: <> + frontend-appname: <> + cf-space: <> + + deploy-infrastructure: + parameters: + tf-path: + type: string + default: ./terraform/dev + cf-password: + type: env_var_name + default: CF_PASSWORD_DEV + cf-username: + type: env_var_name + default: CF_USERNAME_DEV + cf-space: + type: string + default: tanf-dev + cf-org: + type: env_var_name + default: CF_ORG + cf-app: + type: string + default: CF_APP + steps: + - checkout + - run: + name: Install dependencies + command: | + apk update + apk add jq + apk add curl + # TODO: Add Signature check + curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx + mv cf7 /usr/local/bin/cf + - login-cloud-dot-gov: + cf-password: <> + cf-username: <> + cf-space: <> + - run: + name: Export S3 Credentials for TFState + command: | + S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) + { + echo "access_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)\"" + echo "secret_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)\"" + echo "region = \"$(echo "${S3_CREDENTIALS}" | jq -r '.region')\"" + echo "bucket = \"$(echo "${S3_CREDENTIALS}" | jq -r '.bucket')\"" + } >> ./backend_config.tfvars + - run: + name: Prepare Terraform Variables + command: | + S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) + { + echo "cf_password = \"$<>\"" + echo "cf_user = \"$<>\"" + echo "cf_space_name = \"<>\"" + echo "cf_app_name = \"<>\"" + } >> ./variables.tfvars + - terraform/init: + path: <> + backend_config_file: ./backend_config.tfvars + - terraform/validate: + path: <> + - terraform/fmt: + path: <> + - terraform/plan: + path: <> + var_file: ./variables.tfvars + - terraform/apply: + path: <> + var_file: ./variables.tfvars diff --git a/.circleci/config.yml b/.circleci/config.yml index 5db761820..bc37785f8 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,659 +1,13 @@ version: 2.1 -orbs: - node: circleci/node@4.7.0 - terraform: circleci/terraform@2.1.0 - jq: circleci/jq@2.2.0 - -executors: - docker-executor: - docker: - - image: cimg/python:3.10.4 - user: root - machine-executor: - machine: - docker_layer_caching: false - image: ubuntu-2204:2022.10.1 - -commands: - ### - # General re-usable commands - # - cf-check: - steps: - - run: - name: Ensure cf cli is installed, otherwise install it. - command: sudo ./scripts/cf-check.sh - - docker-compose-check: - steps: - - run: - name: Ensure docker-compose exists, otherwise install it. - command: ./scripts/docker-compose-check.sh - - docker-compose-up-backend: - steps: - - run: - name: Build and spin-up Django API service - command: cd tdrs-backend; docker-compose up -d --build - - docker-compose-up-frontend: - steps: - - run: - name: Build and spin-up React application - command: cd tdrs-frontend; docker-compose up -d --build - - disable-npm-audit: - steps: - - run: - name: Disable npm audit warnings in CI - command: npm set audit false - - # This allows us to use the orb stanza for node/install within other commands - # NOTE: This doesn't work correctly on machine executors - install-nodejs: node/install - - # This allows us to use the node orb to install packages within other commands - install-nodejs-packages: node/install-packages - - install-nodejs-machine: - description: | - Installs our target version of Node.JS using NVM (Node Version Manager) - from the install location provided by machine executor images. - steps: - - run: - name: Install Node.JS - command: | - sudo apt-get update - sudo apt-get install -y libgbm-dev - source /opt/circleci/.nvm/nvm.sh - nvm install v16.13 - nvm alias default v16.13 - echo 'export NVM_DIR="/opt/circleci/.nvm"' >> $BASH_ENV - echo "[ -s \"$NVM_DIR/nvm.sh\" ] && . \"$NVM_DIR/nvm.sh\"" >> $BASH_ENV - - run-owasp-scan: - description: Runs OWASP ZAP scanner and stores resulting artifacts. - parameters: - environment: - description: The environment this script is being run for. - type: enum - enum: ["circle", "nightly"] - target: - description: The target application to be scanned. - type: enum - enum: ["backend", "frontend"] - target_env: - description: The target environment to be scanned. - type: enum - default: "develop" - enum: ["staging", "develop", "prod"] - steps: - - run: - name: Execute OWASP ZAP vulnerability scan - #wait up to 60min for scan - no_output_timeout: 60m - command: ./scripts/zap-scanner.sh <> <> <> - - store_artifacts: - path: tdrs-<>/reports/owasp_report.html - - sudo-check: - steps: - - run: - name: Ensure sudo is installed, otherwise install it. - command: ./scripts/sudo-check.sh - - upload-codecov: - description: Uploads testing code coverage results to Codecov - parameters: - component: - description: The component of the application being tested, either backend or frontend. - type: enum - enum: [ "backend", "frontend" ] - coverage-report: - description: The path to the coverage report being uploaded. - type: string - steps: - - run: - name: Ensure Codecov uploader is installed, otherwise install it. - command: ./scripts/codecov-check.sh - - run: - name: Determine Codecov metric flag - command: | - if [ "$CIRCLE_BRANCH" == "main" ] ; then - CURRENT_FLAG=main-<> - elif [ "$CIRCLE_BRANCH" == "master" ] ; then - CURRENT_FLAG=master-<> - else - CURRENT_FLAG=dev-<> - fi - echo "export CURRENT_FLAG=$CURRENT_FLAG" >> $BASH_ENV - - run: - name: Upload code coverage report if target branch - command: codecov -t "$CODECOV_TOKEN" -f <> -F "$CURRENT_FLAG" - - ### - # Deployment commands - # - login-cloud-dot-gov: - description: Authenticates with Cloud.gov and sets org and space targets - parameters: - cf-password: - type: env_var_name - default: CF_PASSWORD_DEV - cf-org: - type: env_var_name - default: CF_ORG - cf-space: - type: string - default: tanf-dev - cf-username: - type: env_var_name - default: CF_USERNAME_DEV - steps: - - run: - name: Login to Cloud.gov and set application targets - command: | - cf login -a https://api.fr.cloud.gov \ - -u ${<>} \ - -p ${<>} \ - -o ${<>} \ - -s <> - get-app-deploy-strategy: - parameters: - appname: - type: string - steps: - - run: - name: Determine deploy strategy - command: | - # NOTE: The || true is a no-op included to suppress exit codes which - # would cause the step to exit early due to use of pipefail - APP_GUID=$(cf app <> --guid || true) - if [ "$APP_GUID" == "FAILED" ]; then - echo "export DEPLOY_STRATEGY=initial" >> $BASH_ENV - else - echo "export DEPLOY_STRATEGY=rolling" >> $BASH_ENV - fi - deploy-backend: - parameters: - backend-appname: - default: tdp-backend - type: string - cf-space: - default: tanf-dev - type: string - steps: - - get-app-deploy-strategy: - appname: <> - - run: - name: Deploy backend application - command: | - bash ./scripts/deploy-backend.sh \ - $DEPLOY_STRATEGY \ - <> \ - <> - deploy-clamav: - parameters: - backend-appname: - default: tdp-backend - type: string - cf-org: - default: CF_ORG - type: env_var_name - cf-space: - default: tanf-dev - type: string - steps: - - run: - name: Deploy ClamAV REST application - command: | - cf push clamav-rest -f tdrs-backend/manifest.clamav.yml \ - --var cf-space=<> \ - - run: - name: Enable internal route between backend and clamav-rest app - command: | - cf add-network-policy <> clamav-rest \ - -s <> \ - -o ${<>} \ - --protocol tcp \ - --port 9000 - deploy-frontend: - parameters: - environment: - description: The environment to deploy to. - type: enum - enum: [ "development", "production" ] - default: development - backend-appname: - default: tdp-backend - type: string - frontend-appname: - default: tdp-frontend - type: string - -# So the frontend knows what space its in for the banner. -# I am unclear if the domain is a reliable metric to make this function -# It seems like it might not be working - cf-space: - default: dev - type: string - steps: - - install-nodejs: - node-version: "16.13" - - disable-npm-audit - - install-nodejs-packages: - app-dir: tdrs-frontend - - get-app-deploy-strategy: - appname: <> - - run: - name: Deploy frontend application - command: | - bash ./scripts/deploy-frontend.sh \ - $DEPLOY_STRATEGY \ - <> \ - <> \ - <> \ - <> - deploy-cloud-dot-gov: - parameters: - environment: - description: The environment to deploy to. - type: enum - enum: [ "development", "production" ] - default: development - backend-appname: - default: tdp-backend - type: string - cf-password: - default: CF_PASSWORD_DEV - type: env_var_name - cf-org: - default: CF_ORG - type: env_var_name - cf-space: - default: tanf-dev - type: string - cf-username: - default: CF_USERNAME_DEV - type: env_var_name - frontend-appname: - default: tdp-frontend - type: string - steps: - - checkout - - sudo-check - - cf-check - - login-cloud-dot-gov: - cf-password: <> - cf-org: <> - cf-space: <> - cf-username: <> - - deploy-backend: - backend-appname: <> - cf-space: <> - - deploy-clamav: - backend-appname: <> - cf-org: <> - cf-space: <> - - deploy-frontend: - environment: <> - backend-appname: <> - frontend-appname: <> - cf-space: <> - - deploy-infrastructure: - parameters: - tf-path: - type: string - default: ./terraform/dev - cf-password: - type: env_var_name - default: CF_PASSWORD_DEV - cf-username: - type: env_var_name - default: CF_USERNAME_DEV - cf-space: - type: string - default: tanf-dev - cf-org: - type: env_var_name - default: CF_ORG - cf-app: - type: string - default: CF_APP - steps: - - checkout - - run: - name: Install dependencies - command: | - apk update - apk add jq - apk add curl - # TODO: Add Signature check - curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx - mv cf7 /usr/local/bin/cf - - login-cloud-dot-gov: - cf-password: <> - cf-username: <> - cf-space: <> - - run: - name: Export S3 Credentials for TFState - command: | - S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) - { - echo "access_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)\"" - echo "secret_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)\"" - echo "region = \"$(echo "${S3_CREDENTIALS}" | jq -r '.region')\"" - echo "bucket = \"$(echo "${S3_CREDENTIALS}" | jq -r '.bucket')\"" - } >> ./backend_config.tfvars - - run: - name: Prepare Terraform Variables - command: | - S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) - { - echo "cf_password = \"$<>\"" - echo "cf_user = \"$<>\"" - echo "cf_space_name = \"<>\"" - echo "cf_app_name = \"<>\"" - } >> ./variables.tfvars - - terraform/init: - path: <> - backend_config_file: ./backend_config.tfvars - - terraform/validate: - path: <> - - terraform/fmt: - path: <> - - terraform/plan: - path: <> - var_file: ./variables.tfvars - - terraform/apply: - path: <> - var_file: ./variables.tfvars - -jobs: - secrets-check: - executor: docker-executor - steps: - - checkout - - run: - name: "git-secrets: Scan repository for committed secrets" - command: ./scripts/git-secrets-check.sh - - run: - name: "trufflehog: Scan repository for committed secrets" - command: ./scripts/trufflehog-check.sh $CIRCLE_BRANCH - - test-backend: - executor: machine-executor - steps: - - checkout - - docker-compose-check - - docker-compose-up-backend - - run: - name: Execute Python Linting Test - command: cd tdrs-backend; docker-compose run --rm web bash -c "flake8 ." - - run: - name: Run Unit Tests And Create Code Coverage Report - command: | - cd tdrs-backend; - docker-compose run --rm web bash -c "./wait_for_services.sh && pytest --cov-report=xml" - - upload-codecov: - component: backend - coverage-report: ./tdrs-backend/coverage.xml +# this allows you to use CircleCI's dynamic configuration feature +setup: true - test-frontend: - executor: machine-executor - working_directory: ~/tdp-apps - steps: - - checkout - - install-nodejs-machine - - disable-npm-audit - - install-nodejs-packages: - app-dir: tdrs-frontend - - run: - name: Run ESLint - command: cd tdrs-frontend; npm run lint - - run: - name: Run Pa11y Accessibility Tests - command: cd tdrs-frontend; mkdir pa11y-screenshots/; npm run test:accessibility - - run: - name: Run Jest Unit Tests - command: cd tdrs-frontend; npm run test:ci - - upload-codecov: - component: frontend - coverage-report: ./tdrs-frontend/coverage/lcov.info - - store_artifacts: - path: tdrs-frontend/pa11y-screenshots/ - - backend-owasp-scan: - executor: machine-executor - working_directory: ~/tdp-apps - steps: - - checkout - - docker-compose-check - - docker-compose-up-backend - - run: - name: Wait for Django to become available - command: | - cd tdrs-backend; - docker-compose run --rm zaproxy bash -c \ - "PATH=$PATH:/home/zap/.local/bin && - pip install wait-for-it && - wait-for-it --service http://web:8080 \ - --timeout 60 \ - -- echo \"Django is ready\"" - - run-owasp-scan: - environment: circle - target: backend - - make_erd: - executor: machine-executor - working_directory: ~/tdp_apps - steps: - - checkout - - docker-compose-check - - run: - name: Run graph_models - command: | - cd tdrs-backend; - docker-compose run --rm web bash -c \ - "./manage.py graph_models -a -g -o tdp_erd.png" - - store_artifacts: - path: tdrs-backend/tdp_erd.png - - frontend-owasp-scan: - executor: machine-executor - working_directory: ~/tdp-apps - steps: - - checkout - - docker-compose-check - - docker-compose-up-frontend - - run: - name: Wait for frontend to become available - command: | - cd tdrs-frontend; - docker-compose run --rm zaproxy bash -c \ - "PATH=$PATH:/home/zap/.local/bin && - pip install wait-for-it && - wait-for-it --service http://tdp-frontend/ \ - --timeout 60 \ - -- echo \"Frontend is ready\"" - - run-owasp-scan: - environment: circle - target: frontend - - nightly-owasp-scan: - executor: machine-executor - working_directory: ~/tdp-apps - parameters: - cf_password: - type: string - default: CF_PASSWORD_STAGING - cf_username: - type: string - default: CF_USERNAME_STAGING - cf_space: - type: string - default: tanf-staging - target_env: - type: enum - enum: [ "staging", "develop", "prod" ] - steps: - - checkout - - sudo-check - - cf-check - - docker-compose-check - - run-owasp-scan: - environment: nightly - target: backend - target_env: <> - - run-owasp-scan: - environment: nightly - target: frontend - target_env: <> - - login-cloud-dot-gov: - cf-password: <> - cf-space: <> - cf-username: <> - - run: - name: Run post-processing task to record OWASP ZAP results - command: | - # Construct the project slug from the current branch name and user - PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME - - # These environment variables are exported to Circle CI's BASH_ENV - # by the zap-scanner.sh script for each respective app target. - CMD_ARGS=( - "$CIRCLE_BUILD_NUM" - --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} - --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} - --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} - --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} - --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} - --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} - --project-slug $PROJECT_SLUG - ) - # Evaluate the full command before passing it in so it doesn't - # get improperly interpolated by Cloud.gov. - CMD="python manage.py process_owasp_scan ${CMD_ARGS[@]}" - # Submit a CF Task for execution that will run the necessary command - cf run-task tdp-backend-<< parameters.target_env >> \ - --command "$CMD" \ - --name nightly-owasp-scan - deploy-infrastructure-dev: - executor: terraform/default - working_directory: ~/tdp-deploy - parameters: - target_env: - type: string - steps: - - deploy-infrastructure: - cf-app: << parameters.target_env >> - - deploy-dev: - executor: docker-executor - parameters: - target_env: - type: string - working_directory: ~/tdp-deploy - steps: - - deploy-cloud-dot-gov: - backend-appname: tdp-backend-<< parameters.target_env >> - frontend-appname: tdp-frontend-<< parameters.target_env >> - - deploy-infrastructure-staging: - executor: terraform/default - working_directory: ~/tdp-deploy - steps: - - deploy-infrastructure: - cf-password: CF_PASSWORD_STAGING - cf-username: CF_USERNAME_STAGING - cf-space: tanf-staging - tf-path: ./terraform/staging - - deploy-staging: - executor: docker-executor - working_directory: ~/tdp-deploy - steps: - - deploy-cloud-dot-gov: - backend-appname: tdp-backend-staging - frontend-appname: tdp-frontend-staging - cf-password: CF_PASSWORD_STAGING - cf-space: tanf-staging - cf-username: CF_USERNAME_STAGING - - deploy-infrastructure-develop: - executor: terraform/default - working_directory: ~/tdp-deploy - steps: - - deploy-infrastructure: - cf-password: CF_PASSWORD_STAGING - cf-username: CF_USERNAME_STAGING - cf-space: tanf-staging - tf-path: ./terraform/staging - - deploy-develop: - executor: docker-executor - working_directory: ~/tdp-deploy - steps: - - deploy-cloud-dot-gov: - backend-appname: tdp-backend-develop - frontend-appname: tdp-frontend-develop - cf-password: CF_PASSWORD_STAGING - cf-space: tanf-staging - cf-username: CF_USERNAME_STAGING - - deploy-project-updates-site: - parameters: - cf-org: - default: CF_ORG - type: env_var_name - cf-space: - default: tanf-dev - type: string - cf-password: - type: env_var_name - default: CF_PASSWORD_DEV - cf-username: - type: env_var_name - default: CF_USERNAME_DEV - executor: docker-executor - working_directory: ~/tdp-deploy - steps: - - checkout - - sudo-check - - cf-check - - login-cloud-dot-gov: - cf-password: <> - cf-org: <> - cf-space: <> - cf-username: <> - - run: - name: Deploy TDP Project Updates Site - command: ./scripts/deploy-tdp-product-update-site.sh rolling tdp-project-updates - - deploy-infrastructure-production: - executor: terraform/default - working_directory: ~/tdp-deploy - steps: - - deploy-infrastructure: - cf-password: CF_PASSWORD_PROD - cf-username: CF_USERNAME_PROD - cf-space: tanf-prod - tf-path: ./terraform/production - - deploy-production: - executor: docker-executor - working_directory: ~/tdp-deploy - steps: - - deploy-cloud-dot-gov: - environment: production - backend-appname: tdp-backend-prod - frontend-appname: tdp-frontend-prod - cf-password: CF_PASSWORD_PROD - cf-space: tanf-prod - cf-username: CF_USERNAME_PROD +# the continuation orb is required in order to use dynamic configuration +orbs: + continuation: circleci/continuation@0.1.2 +# parameters from github actions parameters: run_dev_deployment: type: boolean @@ -667,124 +21,23 @@ parameters: type: string default: '' -workflows: - build-and-test: - unless: - or: - - << pipeline.parameters.run_dev_deployment >> - - << pipeline.parameters.run_owasp_scan >> - jobs: - - secrets-check - - test-frontend: - requires: - - secrets-check - - test-backend: - requires: - - secrets-check - - dev-deployment: - when: - << pipeline.parameters.run_dev_deployment >> - jobs: - - deploy-infrastructure-dev: - target_env: << pipeline.parameters.target_env >> - - deploy-dev: - target_env: << pipeline.parameters.target_env >> - requires: - - deploy-infrastructure-dev - - nightly: - jobs: - - nightly-owasp-scan: - target_env: develop - filters: - branches: - only: - - develop - - nightly-owasp-scan: - target_env: staging - filters: - branches: - only: - - main - - nightly-owasp-scan: - target_env: prod - cf_password: CF_PASSWORD_PROD - cf_username: CF_USERNAME_PROD - cf_space: tanf-prod - filters: - branches: - only: - - master - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - develop - - main - - master - - erd: - jobs: - - make_erd: - filters: - branches: - only: - develop - - owasp-scan: - when: << pipeline.parameters.run_owasp_scan >> - jobs: - - backend-owasp-scan - - frontend-owasp-scan - - staging-deployment: - unless: << pipeline.parameters.run_dev_deployment >> - jobs: - - deploy-project-updates-site: - filters: - branches: - only: - - develop - - deploy-infrastructure-staging: - filters: - branches: - only: - - main - - deploy-staging: - requires: - - deploy-infrastructure-staging - filters: - branches: - only: - - main - - deploy-infrastructure-develop: - filters: - branches: - only: - - develop - - deploy-develop: - requires: - - deploy-infrastructure-develop - filters: - branches: - only: - - develop +# our defined job, and its steps +jobs: + setup: + executor: continuation/default + steps: + - checkout # checkout code + - run: # run a command + name: Generate config + command: | + cd .circleci + chmod +x generate_config.sh + ./generate_config.sh + - continuation/continue: + configuration_path: .circleci/generated_config.yml # use newly generated config to continue - production-deployment: - unless: << pipeline.parameters.run_dev_deployment >> +# our single workflow, that triggers the setup job defined above +workflows: + setup: jobs: - - deploy-infrastructure-production: - filters: - branches: - only: - - master - - deploy-production: - requires: - - deploy-infrastructure-production - filters: - branches: - only: - - master + - setup diff --git a/.circleci/generate_config.sh b/.circleci/generate_config.sh new file mode 100644 index 000000000..00e1ad17b --- /dev/null +++ b/.circleci/generate_config.sh @@ -0,0 +1,8 @@ +#!/usr/bin/sh + +cat base_config.yml > generated_config.yml +cat commands.yml >> generated_config.yml +cat jobs.yml >> generated_config.yml +cat workflows.yml >> generated_config.yml + +cat generated_config.yml diff --git a/.circleci/jobs.yml b/.circleci/jobs.yml new file mode 100644 index 000000000..af93d9ef8 --- /dev/null +++ b/.circleci/jobs.yml @@ -0,0 +1,282 @@ +jobs: + secrets-check: + executor: docker-executor + steps: + - checkout + - run: + name: "git-secrets: Scan repository for committed secrets" + command: ./scripts/git-secrets-check.sh + - run: + name: "trufflehog: Scan repository for committed secrets" + command: ./scripts/trufflehog-check.sh $CIRCLE_BRANCH + + test-backend: + executor: machine-executor + steps: + - checkout + - docker-compose-check + - docker-compose-up-backend + - run: + name: Execute Python Linting Test + command: cd tdrs-backend; docker-compose run --rm web bash -c "flake8 ." + - run: + name: Run Unit Tests And Create Code Coverage Report + command: | + cd tdrs-backend; + docker-compose run --rm web bash -c "./wait_for_services.sh && pytest --cov-report=xml" + - upload-codecov: + component: backend + coverage-report: ./tdrs-backend/coverage.xml + + test-frontend: + executor: machine-executor + working_directory: ~/tdp-apps + steps: + - checkout + - install-nodejs-machine + - disable-npm-audit + - install-nodejs-packages: + app-dir: tdrs-frontend + - run: + name: Run ESLint + command: cd tdrs-frontend; npm run lint + - run: + name: Run Pa11y Accessibility Tests + command: cd tdrs-frontend; mkdir pa11y-screenshots/; npm run test:accessibility + - run: + name: Run Jest Unit Tests + command: cd tdrs-frontend; npm run test:ci + - upload-codecov: + component: frontend + coverage-report: ./tdrs-frontend/coverage/lcov.info + - store_artifacts: + path: tdrs-frontend/pa11y-screenshots/ + + backend-owasp-scan: + executor: machine-executor + working_directory: ~/tdp-apps + steps: + - checkout + - docker-compose-check + - docker-compose-up-backend + - run: + name: Wait for Django to become available + command: | + cd tdrs-backend; + docker-compose run --rm zaproxy bash -c \ + "PATH=$PATH:/home/zap/.local/bin && + pip install wait-for-it && + wait-for-it --service http://web:8080 \ + --timeout 60 \ + -- echo \"Django is ready\"" + - run-owasp-scan: + environment: circle + target: backend + + make_erd: + executor: machine-executor + working_directory: ~/tdp_apps + steps: + - checkout + - docker-compose-check + - run: + name: Run graph_models + command: | + cd tdrs-backend; + docker-compose run --rm web bash -c \ + "./manage.py graph_models -a -g -o tdp_erd.png" + - store_artifacts: + path: tdrs-backend/tdp_erd.png + + frontend-owasp-scan: + executor: machine-executor + working_directory: ~/tdp-apps + steps: + - checkout + - docker-compose-check + - docker-compose-up-frontend + - run: + name: Wait for frontend to become available + command: | + cd tdrs-frontend; + docker-compose run --rm zaproxy bash -c \ + "PATH=$PATH:/home/zap/.local/bin && + pip install wait-for-it && + wait-for-it --service http://tdp-frontend/ \ + --timeout 60 \ + -- echo \"Frontend is ready\"" + - run-owasp-scan: + environment: circle + target: frontend + + nightly-owasp-scan: + executor: machine-executor + working_directory: ~/tdp-apps + parameters: + cf_password: + type: string + default: CF_PASSWORD_STAGING + cf_username: + type: string + default: CF_USERNAME_STAGING + cf_space: + type: string + default: tanf-staging + target_env: + type: enum + enum: [ "staging", "develop", "prod" ] + steps: + - checkout + - sudo-check + - cf-check + - docker-compose-check + - run-owasp-scan: + environment: nightly + target: backend + target_env: <> + - run-owasp-scan: + environment: nightly + target: frontend + target_env: <> + - login-cloud-dot-gov: + cf-password: <> + cf-space: <> + cf-username: <> + - run: + name: Run post-processing task to record OWASP ZAP results + command: | + # Construct the project slug from the current branch name and user + PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME + + # These environment variables are exported to Circle CI's BASH_ENV + # by the zap-scanner.sh script for each respective app target. + CMD_ARGS=( + "$CIRCLE_BUILD_NUM" + --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} + --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} + --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} + --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} + --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} + --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} + --project-slug $PROJECT_SLUG + ) + # Evaluate the full command before passing it in so it doesn't + # get improperly interpolated by Cloud.gov. + CMD="python manage.py process_owasp_scan ${CMD_ARGS[@]}" + # Submit a CF Task for execution that will run the necessary command + cf run-task tdp-backend-<< parameters.target_env >> \ + --command "$CMD" \ + --name nightly-owasp-scan + deploy-infrastructure-dev: + executor: terraform/default + working_directory: ~/tdp-deploy + parameters: + target_env: + type: string + steps: + - deploy-infrastructure: + cf-app: << parameters.target_env >> + + deploy-dev: + executor: docker-executor + parameters: + target_env: + type: string + working_directory: ~/tdp-deploy + steps: + - deploy-cloud-dot-gov: + backend-appname: tdp-backend-<< parameters.target_env >> + frontend-appname: tdp-frontend-<< parameters.target_env >> + + deploy-infrastructure-staging: + executor: terraform/default + working_directory: ~/tdp-deploy + steps: + - deploy-infrastructure: + cf-password: CF_PASSWORD_STAGING + cf-username: CF_USERNAME_STAGING + cf-space: tanf-staging + tf-path: ./terraform/staging + + deploy-staging: + executor: docker-executor + working_directory: ~/tdp-deploy + steps: + - deploy-cloud-dot-gov: + backend-appname: tdp-backend-staging + frontend-appname: tdp-frontend-staging + cf-password: CF_PASSWORD_STAGING + cf-space: tanf-staging + cf-username: CF_USERNAME_STAGING + + deploy-infrastructure-develop: + executor: terraform/default + working_directory: ~/tdp-deploy + steps: + - deploy-infrastructure: + cf-password: CF_PASSWORD_STAGING + cf-username: CF_USERNAME_STAGING + cf-space: tanf-staging + tf-path: ./terraform/staging + + deploy-develop: + executor: docker-executor + working_directory: ~/tdp-deploy + steps: + - deploy-cloud-dot-gov: + backend-appname: tdp-backend-develop + frontend-appname: tdp-frontend-develop + cf-password: CF_PASSWORD_STAGING + cf-space: tanf-staging + cf-username: CF_USERNAME_STAGING + + deploy-project-updates-site: + parameters: + cf-org: + default: CF_ORG + type: env_var_name + cf-space: + default: tanf-dev + type: string + cf-password: + type: env_var_name + default: CF_PASSWORD_DEV + cf-username: + type: env_var_name + default: CF_USERNAME_DEV + executor: docker-executor + working_directory: ~/tdp-deploy + steps: + - checkout + - sudo-check + - cf-check + - login-cloud-dot-gov: + cf-password: <> + cf-org: <> + cf-space: <> + cf-username: <> + - run: + name: Deploy TDP Project Updates Site + command: ./scripts/deploy-tdp-product-update-site.sh rolling tdp-project-updates + + deploy-infrastructure-production: + executor: terraform/default + working_directory: ~/tdp-deploy + steps: + - deploy-infrastructure: + cf-password: CF_PASSWORD_PROD + cf-username: CF_USERNAME_PROD + cf-space: tanf-prod + tf-path: ./terraform/production + + deploy-production: + executor: docker-executor + working_directory: ~/tdp-deploy + steps: + - deploy-cloud-dot-gov: + environment: production + backend-appname: tdp-backend-prod + frontend-appname: tdp-frontend-prod + cf-password: CF_PASSWORD_PROD + cf-space: tanf-prod + cf-username: CF_USERNAME_PROD diff --git a/.circleci/workflows.yml b/.circleci/workflows.yml new file mode 100644 index 000000000..b39046ac7 --- /dev/null +++ b/.circleci/workflows.yml @@ -0,0 +1,121 @@ +workflows: + build-and-test: + unless: + or: + - << pipeline.parameters.run_dev_deployment >> + - << pipeline.parameters.run_owasp_scan >> + jobs: + - secrets-check + - test-frontend: + requires: + - secrets-check + - test-backend: + requires: + - secrets-check + + dev-deployment: + when: + << pipeline.parameters.run_dev_deployment >> + jobs: + - deploy-infrastructure-dev: + target_env: << pipeline.parameters.target_env >> + - deploy-dev: + target_env: << pipeline.parameters.target_env >> + requires: + - deploy-infrastructure-dev + + nightly: + jobs: + - nightly-owasp-scan: + target_env: develop + filters: + branches: + only: + - develop + - nightly-owasp-scan: + target_env: staging + filters: + branches: + only: + - main + - nightly-owasp-scan: + target_env: prod + cf_password: CF_PASSWORD_PROD + cf_username: CF_USERNAME_PROD + cf_space: tanf-prod + filters: + branches: + only: + - master + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - develop + - main + - master + + erd: + jobs: + - make_erd: + filters: + branches: + only: + develop + + owasp-scan: + when: << pipeline.parameters.run_owasp_scan >> + jobs: + - backend-owasp-scan + - frontend-owasp-scan + + staging-deployment: + unless: << pipeline.parameters.run_dev_deployment >> + jobs: + - deploy-project-updates-site: + filters: + branches: + only: + - develop + - deploy-infrastructure-staging: + filters: + branches: + only: + - main + - deploy-staging: + requires: + - deploy-infrastructure-staging + filters: + branches: + only: + - main + - deploy-infrastructure-develop: + filters: + branches: + only: + - develop + - deploy-develop: + requires: + - deploy-infrastructure-develop + filters: + branches: + only: + - develop + + production-deployment: + unless: << pipeline.parameters.run_dev_deployment >> + jobs: + - deploy-infrastructure-production: + filters: + branches: + only: + - master + - deploy-production: + requires: + - deploy-infrastructure-production + filters: + branches: + only: + - master From 6f121b70534296d39dd6c781e599e35f537d95d2 Mon Sep 17 00:00:00 2001 From: Cameron Smart Date: Mon, 14 Nov 2022 12:36:39 -0800 Subject: [PATCH 2/4] Update Readme --- .circleci/config.yml | 4 +++- README.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bc37785f8..bbf5c294b 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,9 @@ parameters: type: string default: '' -# our defined job, and its steps +# To help debugging circleci config changes you may want +# to run the generate_config.sh script locally and +# then copy the contents from generated_config.yml into the config.yml file. jobs: setup: executor: continuation/default diff --git a/README.md b/README.md index 5f12aec65..ba9c0a768 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ See [Architecture Decision Record 003 - Application Hosting](./docs/Technical-Do #### Continuous Integration (CI) -On each git push and merge, a comprehensive list of automated checks are run: Unit tests ([Jest](https://jestjs.io/), Linting tests ([ESLint](https://eslint.org/), Accessibility tests ([Pa11y](https://pa11y.org/)), and Security Scanning ([OWASP ZAP](https://owasp.org/www-project-zap/)). The configurations for CI are kept in [`.circleci/config.yml`](https://github.com/HHS/TANF-app/blob/master/.circleci/config.yml). +On each git push and merge, a comprehensive list of automated checks are run: Unit tests ([Jest](https://jestjs.io/), Linting tests ([ESLint](https://eslint.org/), Accessibility tests ([Pa11y](https://pa11y.org/)), and Security Scanning ([OWASP ZAP](https://owasp.org/www-project-zap/)). The configurations for CI are generated by [`.circleci/config.yml`](https://github.com/HHS/TANF-app/blob/master/.circleci/config.yml). Circle Ci workflows, jobs, and commands are separated into respective yaml files. See [Architecture Decision Record 006 - Continuous integration](./docs/Technical-Documentation/Architecture-Decision-Record/006-continuous-integration.md) and [TDP's CircleCi Workflows, Environment Variables, and Builds](./docs/Technical-Documentation/circle-ci.md)- for more details. From 84ff7f7dda41f8d6821fd4cde046d89a4a282919 Mon Sep 17 00:00:00 2001 From: Cameron Smart Date: Thu, 17 Nov 2022 14:23:04 -0800 Subject: [PATCH 3/4] reorder by stage --- .circleci/build-and-test/commands.yml | 66 +++++ .circleci/build-and-test/jobs.yml | 53 ++++ .circleci/commands.yml | 355 -------------------------- .circleci/deployment/commands.yml | 146 +++++++++++ .circleci/deployment/jobs.yml | 45 ++++ .circleci/generate_config.sh | 22 +- .circleci/infrastructure/commands.yml | 95 +++++++ .circleci/infrastructure/jobs.yml | 69 +++++ .circleci/jobs.yml | 282 -------------------- .circleci/owasp/commands.yml | 31 +++ .circleci/owasp/jobs.yml | 101 ++++++++ .circleci/util/commands.yml | 16 ++ .circleci/util/jobs.yml | 15 ++ .circleci/workflows.yml | 95 +++---- 14 files changed, 704 insertions(+), 687 deletions(-) create mode 100644 .circleci/build-and-test/commands.yml create mode 100644 .circleci/build-and-test/jobs.yml delete mode 100644 .circleci/commands.yml create mode 100644 .circleci/deployment/commands.yml create mode 100644 .circleci/deployment/jobs.yml create mode 100644 .circleci/infrastructure/commands.yml create mode 100644 .circleci/infrastructure/jobs.yml delete mode 100644 .circleci/jobs.yml create mode 100644 .circleci/owasp/commands.yml create mode 100644 .circleci/owasp/jobs.yml create mode 100644 .circleci/util/commands.yml create mode 100644 .circleci/util/jobs.yml diff --git a/.circleci/build-and-test/commands.yml b/.circleci/build-and-test/commands.yml new file mode 100644 index 000000000..d96dea2d1 --- /dev/null +++ b/.circleci/build-and-test/commands.yml @@ -0,0 +1,66 @@ +# commands: + docker-compose-check: + steps: + - run: + name: Ensure docker-compose exists, otherwise install it. + command: ./scripts/docker-compose-check.sh + + docker-compose-up-backend: + steps: + - run: + name: Build and spin-up Django API service + command: cd tdrs-backend; docker-compose up -d --build + + upload-codecov: + description: Uploads testing code coverage results to Codecov + parameters: + component: + description: The component of the application being tested, either backend or frontend. + type: enum + enum: [ "backend", "frontend" ] + coverage-report: + description: The path to the coverage report being uploaded. + type: string + steps: + - run: + name: Ensure Codecov uploader is installed, otherwise install it. + command: ./scripts/codecov-check.sh + - run: + name: Determine Codecov metric flag + command: | + if [ "$CIRCLE_BRANCH" == "main" ] ; then + CURRENT_FLAG=main-<> + elif [ "$CIRCLE_BRANCH" == "master" ] ; then + CURRENT_FLAG=master-<> + else + CURRENT_FLAG=dev-<> + fi + echo "export CURRENT_FLAG=$CURRENT_FLAG" >> $BASH_ENV + - run: + name: Upload code coverage report if target branch + command: codecov -t "$CODECOV_TOKEN" -f <> -F "$CURRENT_FLAG" + + install-nodejs-machine: + description: | + Installs our target version of Node.JS using NVM (Node Version Manager) + from the install location provided by machine executor images. + steps: + - run: + name: Install Node.JS + command: | + sudo apt-get update + sudo apt-get install -y libgbm-dev + source /opt/circleci/.nvm/nvm.sh + nvm install v16.13 + nvm alias default v16.13 + echo 'export NVM_DIR="/opt/circleci/.nvm"' >> $BASH_ENV + echo "[ -s \"$NVM_DIR/nvm.sh\" ] && . \"$NVM_DIR/nvm.sh\"" >> $BASH_ENV + + disable-npm-audit: + steps: + - run: + name: Disable npm audit warnings in CI + command: npm set audit false + + # This allows us to use the node orb to install packages within other commands + install-nodejs-packages: node/install-packages diff --git a/.circleci/build-and-test/jobs.yml b/.circleci/build-and-test/jobs.yml new file mode 100644 index 000000000..603af1682 --- /dev/null +++ b/.circleci/build-and-test/jobs.yml @@ -0,0 +1,53 @@ +# jobs: + test-backend: + executor: machine-executor + steps: + - checkout + - docker-compose-check + - docker-compose-up-backend + - run: + name: Execute Python Linting Test + command: cd tdrs-backend; docker-compose run --rm web bash -c "flake8 ." + - run: + name: Run Unit Tests And Create Code Coverage Report + command: | + cd tdrs-backend; + docker-compose run --rm web bash -c "./wait_for_services.sh && pytest --cov-report=xml" + - upload-codecov: + component: backend + coverage-report: ./tdrs-backend/coverage.xml + + test-frontend: + executor: machine-executor + working_directory: ~/tdp-apps + steps: + - checkout + - install-nodejs-machine + - disable-npm-audit + - install-nodejs-packages: + app-dir: tdrs-frontend + - run: + name: Run ESLint + command: cd tdrs-frontend; npm run lint + - run: + name: Run Pa11y Accessibility Tests + command: cd tdrs-frontend; mkdir pa11y-screenshots/; npm run test:accessibility + - run: + name: Run Jest Unit Tests + command: cd tdrs-frontend; npm run test:ci + - upload-codecov: + component: frontend + coverage-report: ./tdrs-frontend/coverage/lcov.info + - store_artifacts: + path: tdrs-frontend/pa11y-screenshots/ + + secrets-check: + executor: docker-executor + steps: + - checkout + - run: + name: "git-secrets: Scan repository for committed secrets" + command: ./scripts/git-secrets-check.sh + - run: + name: "trufflehog: Scan repository for committed secrets" + command: ./scripts/trufflehog-check.sh $CIRCLE_BRANCH diff --git a/.circleci/commands.yml b/.circleci/commands.yml deleted file mode 100644 index f7ab324e1..000000000 --- a/.circleci/commands.yml +++ /dev/null @@ -1,355 +0,0 @@ -commands: - ### - # General re-usable commands - # - cf-check: - steps: - - run: - name: Ensure cf cli is installed, otherwise install it. - command: sudo ./scripts/cf-check.sh - - docker-compose-check: - steps: - - run: - name: Ensure docker-compose exists, otherwise install it. - command: ./scripts/docker-compose-check.sh - - docker-compose-up-backend: - steps: - - run: - name: Build and spin-up Django API service - command: cd tdrs-backend; docker-compose up -d --build - - docker-compose-up-frontend: - steps: - - run: - name: Build and spin-up React application - command: cd tdrs-frontend; docker-compose up -d --build - - disable-npm-audit: - steps: - - run: - name: Disable npm audit warnings in CI - command: npm set audit false - - # This allows us to use the orb stanza for node/install within other commands - # NOTE: This doesn't work correctly on machine executors - install-nodejs: node/install - - # This allows us to use the node orb to install packages within other commands - install-nodejs-packages: node/install-packages - - install-nodejs-machine: - description: | - Installs our target version of Node.JS using NVM (Node Version Manager) - from the install location provided by machine executor images. - steps: - - run: - name: Install Node.JS - command: | - sudo apt-get update - sudo apt-get install -y libgbm-dev - source /opt/circleci/.nvm/nvm.sh - nvm install v16.13 - nvm alias default v16.13 - echo 'export NVM_DIR="/opt/circleci/.nvm"' >> $BASH_ENV - echo "[ -s \"$NVM_DIR/nvm.sh\" ] && . \"$NVM_DIR/nvm.sh\"" >> $BASH_ENV - - run-owasp-scan: - description: Runs OWASP ZAP scanner and stores resulting artifacts. - parameters: - environment: - description: The environment this script is being run for. - type: enum - enum: ["circle", "nightly"] - target: - description: The target application to be scanned. - type: enum - enum: ["backend", "frontend"] - target_env: - description: The target environment to be scanned. - type: enum - default: "develop" - enum: ["staging", "develop", "prod"] - steps: - - run: - name: Execute OWASP ZAP vulnerability scan - #wait up to 60min for scan - no_output_timeout: 60m - command: ./scripts/zap-scanner.sh <> <> <> - - store_artifacts: - path: tdrs-<>/reports/owasp_report.html - - sudo-check: - steps: - - run: - name: Ensure sudo is installed, otherwise install it. - command: ./scripts/sudo-check.sh - - upload-codecov: - description: Uploads testing code coverage results to Codecov - parameters: - component: - description: The component of the application being tested, either backend or frontend. - type: enum - enum: [ "backend", "frontend" ] - coverage-report: - description: The path to the coverage report being uploaded. - type: string - steps: - - run: - name: Ensure Codecov uploader is installed, otherwise install it. - command: ./scripts/codecov-check.sh - - run: - name: Determine Codecov metric flag - command: | - if [ "$CIRCLE_BRANCH" == "main" ] ; then - CURRENT_FLAG=main-<> - elif [ "$CIRCLE_BRANCH" == "master" ] ; then - CURRENT_FLAG=master-<> - else - CURRENT_FLAG=dev-<> - fi - echo "export CURRENT_FLAG=$CURRENT_FLAG" >> $BASH_ENV - - run: - name: Upload code coverage report if target branch - command: codecov -t "$CODECOV_TOKEN" -f <> -F "$CURRENT_FLAG" - - ### - # Deployment commands - # - login-cloud-dot-gov: - description: Authenticates with Cloud.gov and sets org and space targets - parameters: - cf-password: - type: env_var_name - default: CF_PASSWORD_DEV - cf-org: - type: env_var_name - default: CF_ORG - cf-space: - type: string - default: tanf-dev - cf-username: - type: env_var_name - default: CF_USERNAME_DEV - steps: - - run: - name: Login to Cloud.gov and set application targets - command: | - cf login -a https://api.fr.cloud.gov \ - -u ${<>} \ - -p ${<>} \ - -o ${<>} \ - -s <> - get-app-deploy-strategy: - parameters: - appname: - type: string - steps: - - run: - name: Determine deploy strategy - command: | - # NOTE: The || true is a no-op included to suppress exit codes which - # would cause the step to exit early due to use of pipefail - APP_GUID=$(cf app <> --guid || true) - if [ "$APP_GUID" == "FAILED" ]; then - echo "export DEPLOY_STRATEGY=initial" >> $BASH_ENV - else - echo "export DEPLOY_STRATEGY=rolling" >> $BASH_ENV - fi - deploy-backend: - parameters: - backend-appname: - default: tdp-backend - type: string - cf-space: - default: tanf-dev - type: string - steps: - - get-app-deploy-strategy: - appname: <> - - run: - name: Deploy backend application - command: | - bash ./scripts/deploy-backend.sh \ - $DEPLOY_STRATEGY \ - <> \ - <> - deploy-clamav: - parameters: - backend-appname: - default: tdp-backend - type: string - cf-org: - default: CF_ORG - type: env_var_name - cf-space: - default: tanf-dev - type: string - steps: - - run: - name: Deploy ClamAV REST application - command: | - cf push clamav-rest -f tdrs-backend/manifest.clamav.yml \ - --var cf-space=<> \ - - run: - name: Enable internal route between backend and clamav-rest app - command: | - cf add-network-policy <> clamav-rest \ - -s <> \ - -o ${<>} \ - --protocol tcp \ - --port 9000 - deploy-frontend: - parameters: - environment: - description: The environment to deploy to. - type: enum - enum: [ "development", "production" ] - default: development - backend-appname: - default: tdp-backend - type: string - frontend-appname: - default: tdp-frontend - type: string - -# So the frontend knows what space its in for the banner. -# I am unclear if the domain is a reliable metric to make this function -# It seems like it might not be working - cf-space: - default: dev - type: string - steps: - - install-nodejs: - node-version: "16.13" - - disable-npm-audit - - install-nodejs-packages: - app-dir: tdrs-frontend - - get-app-deploy-strategy: - appname: <> - - run: - name: Deploy frontend application - command: | - bash ./scripts/deploy-frontend.sh \ - $DEPLOY_STRATEGY \ - <> \ - <> \ - <> \ - <> - deploy-cloud-dot-gov: - parameters: - environment: - description: The environment to deploy to. - type: enum - enum: [ "development", "production" ] - default: development - backend-appname: - default: tdp-backend - type: string - cf-password: - default: CF_PASSWORD_DEV - type: env_var_name - cf-org: - default: CF_ORG - type: env_var_name - cf-space: - default: tanf-dev - type: string - cf-username: - default: CF_USERNAME_DEV - type: env_var_name - frontend-appname: - default: tdp-frontend - type: string - steps: - - checkout - - sudo-check - - cf-check - - login-cloud-dot-gov: - cf-password: <> - cf-org: <> - cf-space: <> - cf-username: <> - - deploy-backend: - backend-appname: <> - cf-space: <> - - deploy-clamav: - backend-appname: <> - cf-org: <> - cf-space: <> - - deploy-frontend: - environment: <> - backend-appname: <> - frontend-appname: <> - cf-space: <> - - deploy-infrastructure: - parameters: - tf-path: - type: string - default: ./terraform/dev - cf-password: - type: env_var_name - default: CF_PASSWORD_DEV - cf-username: - type: env_var_name - default: CF_USERNAME_DEV - cf-space: - type: string - default: tanf-dev - cf-org: - type: env_var_name - default: CF_ORG - cf-app: - type: string - default: CF_APP - steps: - - checkout - - run: - name: Install dependencies - command: | - apk update - apk add jq - apk add curl - # TODO: Add Signature check - curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx - mv cf7 /usr/local/bin/cf - - login-cloud-dot-gov: - cf-password: <> - cf-username: <> - cf-space: <> - - run: - name: Export S3 Credentials for TFState - command: | - S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) - { - echo "access_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)\"" - echo "secret_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)\"" - echo "region = \"$(echo "${S3_CREDENTIALS}" | jq -r '.region')\"" - echo "bucket = \"$(echo "${S3_CREDENTIALS}" | jq -r '.bucket')\"" - } >> ./backend_config.tfvars - - run: - name: Prepare Terraform Variables - command: | - S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) - { - echo "cf_password = \"$<>\"" - echo "cf_user = \"$<>\"" - echo "cf_space_name = \"<>\"" - echo "cf_app_name = \"<>\"" - } >> ./variables.tfvars - - terraform/init: - path: <> - backend_config_file: ./backend_config.tfvars - - terraform/validate: - path: <> - - terraform/fmt: - path: <> - - terraform/plan: - path: <> - var_file: ./variables.tfvars - - terraform/apply: - path: <> - var_file: ./variables.tfvars diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml new file mode 100644 index 000000000..97015e18b --- /dev/null +++ b/.circleci/deployment/commands.yml @@ -0,0 +1,146 @@ +# commands: + deploy-cloud-dot-gov: + parameters: + environment: + description: The environment to deploy to. + type: enum + enum: [ "development", "production" ] + default: development + backend-appname: + default: tdp-backend + type: string + cf-password: + default: CF_PASSWORD_DEV + type: env_var_name + cf-org: + default: CF_ORG + type: env_var_name + cf-space: + default: tanf-dev + type: string + cf-username: + default: CF_USERNAME_DEV + type: env_var_name + frontend-appname: + default: tdp-frontend + type: string + steps: + - checkout + - sudo-check + - cf-check + - login-cloud-dot-gov: + cf-password: <> + cf-org: <> + cf-space: <> + cf-username: <> + - deploy-backend: + backend-appname: <> + cf-space: <> + - deploy-clamav: + backend-appname: <> + cf-org: <> + cf-space: <> + - deploy-frontend: + environment: <> + backend-appname: <> + frontend-appname: <> + cf-space: <> + + deploy-backend: + parameters: + backend-appname: + default: tdp-backend + type: string + cf-space: + default: tanf-dev + type: string + steps: + - get-app-deploy-strategy: + appname: <> + - run: + name: Deploy backend application + command: | + bash ./scripts/deploy-backend.sh \ + $DEPLOY_STRATEGY \ + <> \ + <> + + deploy-clamav: + parameters: + backend-appname: + default: tdp-backend + type: string + cf-org: + default: CF_ORG + type: env_var_name + cf-space: + default: tanf-dev + type: string + steps: + - run: + name: Deploy ClamAV REST application + command: | + cf push clamav-rest -f tdrs-backend/manifest.clamav.yml \ + --var cf-space=<> \ + - run: + name: Enable internal route between backend and clamav-rest app + command: | + cf add-network-policy <> clamav-rest \ + -s <> \ + -o ${<>} \ + --protocol tcp \ + --port 9000 + + deploy-frontend: + parameters: + environment: + description: The environment to deploy to. + type: enum + enum: [ "development", "production" ] + default: development + backend-appname: + default: tdp-backend + type: string + frontend-appname: + default: tdp-frontend + type: string +# So the frontend knows what space its in for the banner. +# I am unclear if the domain is a reliable metric to make this function +# It seems like it might not be working + cf-space: + default: dev + type: string + steps: + - install-nodejs: + node-version: "16.13" + - disable-npm-audit + - install-nodejs-packages: + app-dir: tdrs-frontend + - get-app-deploy-strategy: + appname: <> + - run: + name: Deploy frontend application + command: | + bash ./scripts/deploy-frontend.sh \ + $DEPLOY_STRATEGY \ + <> \ + <> \ + <> \ + <> + + get-app-deploy-strategy: + parameters: + appname: + type: string + steps: + - run: + name: Determine deploy strategy + command: | + # NOTE: The || true is a no-op included to suppress exit codes which + # would cause the step to exit early due to use of pipefail + APP_GUID=$(cf app <> --guid || true) + if [ "$APP_GUID" == "FAILED" ]; then + echo "export DEPLOY_STRATEGY=initial" >> $BASH_ENV + else + echo "export DEPLOY_STRATEGY=rolling" >> $BASH_ENV + fi diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml new file mode 100644 index 000000000..cff865124 --- /dev/null +++ b/.circleci/deployment/jobs.yml @@ -0,0 +1,45 @@ +# jobs: + deploy-dev: + executor: docker-executor + parameters: + target_env: + type: string + working_directory: ~/tdp-deploy + steps: + - deploy-cloud-dot-gov: + backend-appname: tdp-backend-<< parameters.target_env >> + frontend-appname: tdp-frontend-<< parameters.target_env >> + + deploy-staging: + executor: docker-executor + working_directory: ~/tdp-deploy + steps: + - deploy-cloud-dot-gov: + backend-appname: tdp-backend-staging + frontend-appname: tdp-frontend-staging + cf-password: CF_PASSWORD_STAGING + cf-space: tanf-staging + cf-username: CF_USERNAME_STAGING + + deploy-develop: + executor: docker-executor + working_directory: ~/tdp-deploy + steps: + - deploy-cloud-dot-gov: + backend-appname: tdp-backend-develop + frontend-appname: tdp-frontend-develop + cf-password: CF_PASSWORD_STAGING + cf-space: tanf-staging + cf-username: CF_USERNAME_STAGING + + deploy-production: + executor: docker-executor + working_directory: ~/tdp-deploy + steps: + - deploy-cloud-dot-gov: + environment: production + backend-appname: tdp-backend-prod + frontend-appname: tdp-frontend-prod + cf-password: CF_PASSWORD_PROD + cf-space: tanf-prod + cf-username: CF_USERNAME_PROD diff --git a/.circleci/generate_config.sh b/.circleci/generate_config.sh index 00e1ad17b..4caf1a80e 100644 --- a/.circleci/generate_config.sh +++ b/.circleci/generate_config.sh @@ -1,8 +1,24 @@ #!/usr/bin/sh cat base_config.yml > generated_config.yml -cat commands.yml >> generated_config.yml -cat jobs.yml >> generated_config.yml + +# Commands +echo 'commands:' >> generated_config.yml +cat build-and-test/commands.yml >> generated_config.yml +cat infrastructure/commands.yml >> generated_config.yml +cat deployment/commands.yml >> generated_config.yml +cat owasp/commands.yml >> generated_config.yml +cat util/commands.yml >> generated_config.yml + +# Jobs +echo 'jobs:' >> generated_config.yml +cat build-and-test/jobs.yml >> generated_config.yml +cat infrastructure/jobs.yml >> generated_config.yml +cat deployment/jobs.yml >> generated_config.yml +cat owasp/jobs.yml >> generated_config.yml +cat util/jobs.yml >> generated_config.yml + +# Workflows cat workflows.yml >> generated_config.yml -cat generated_config.yml +cat generated_config.yml \ No newline at end of file diff --git a/.circleci/infrastructure/commands.yml b/.circleci/infrastructure/commands.yml new file mode 100644 index 000000000..5dc6e6114 --- /dev/null +++ b/.circleci/infrastructure/commands.yml @@ -0,0 +1,95 @@ +# commands: + deploy-infrastructure: + parameters: + tf-path: + type: string + default: ./terraform/dev + cf-password: + type: env_var_name + default: CF_PASSWORD_DEV + cf-username: + type: env_var_name + default: CF_USERNAME_DEV + cf-space: + type: string + default: tanf-dev + cf-org: + type: env_var_name + default: CF_ORG + cf-app: + type: string + default: CF_APP + steps: + - checkout + - run: + name: Install dependencies + command: | + apk update + apk add jq + apk add curl + # TODO: Add Signature check + curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx + mv cf7 /usr/local/bin/cf + - login-cloud-dot-gov: + cf-password: <> + cf-username: <> + cf-space: <> + - run: + name: Export S3 Credentials for TFState + command: | + S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) + { + echo "access_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)\"" + echo "secret_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)\"" + echo "region = \"$(echo "${S3_CREDENTIALS}" | jq -r '.region')\"" + echo "bucket = \"$(echo "${S3_CREDENTIALS}" | jq -r '.bucket')\"" + } >> ./backend_config.tfvars + - run: + name: Prepare Terraform Variables + command: | + S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) + { + echo "cf_password = \"$<>\"" + echo "cf_user = \"$<>\"" + echo "cf_space_name = \"<>\"" + echo "cf_app_name = \"<>\"" + } >> ./variables.tfvars + - terraform/init: + path: <> + backend_config_file: ./backend_config.tfvars + - terraform/validate: + path: <> + - terraform/fmt: + path: <> + - terraform/plan: + path: <> + var_file: ./variables.tfvars + - terraform/apply: + path: <> + var_file: ./variables.tfvars + + login-cloud-dot-gov: + description: Authenticates with Cloud.gov and sets org and space targets + parameters: + cf-password: + type: env_var_name + default: CF_PASSWORD_DEV + cf-org: + type: env_var_name + default: CF_ORG + cf-space: + type: string + default: tanf-dev + cf-username: + type: env_var_name + default: CF_USERNAME_DEV + steps: + - run: + name: Login to Cloud.gov and set application targets + command: | + cf login -a https://api.fr.cloud.gov \ + -u ${<>} \ + -p ${<>} \ + -o ${<>} \ + -s <> + \ No newline at end of file diff --git a/.circleci/infrastructure/jobs.yml b/.circleci/infrastructure/jobs.yml new file mode 100644 index 000000000..845c20528 --- /dev/null +++ b/.circleci/infrastructure/jobs.yml @@ -0,0 +1,69 @@ +# jobs: + deploy-infrastructure-dev: + executor: terraform/default + working_directory: ~/tdp-deploy + parameters: + target_env: + type: string + steps: + - deploy-infrastructure: + cf-app: << parameters.target_env >> + + deploy-infrastructure-staging: + executor: terraform/default + working_directory: ~/tdp-deploy + steps: + - deploy-infrastructure: + cf-password: CF_PASSWORD_STAGING + cf-username: CF_USERNAME_STAGING + cf-space: tanf-staging + tf-path: ./terraform/staging + + deploy-infrastructure-develop: + executor: terraform/default + working_directory: ~/tdp-deploy + steps: + - deploy-infrastructure: + cf-password: CF_PASSWORD_STAGING + cf-username: CF_USERNAME_STAGING + cf-space: tanf-staging + tf-path: ./terraform/staging + + deploy-infrastructure-production: + executor: terraform/default + working_directory: ~/tdp-deploy + steps: + - deploy-infrastructure: + cf-password: CF_PASSWORD_PROD + cf-username: CF_USERNAME_PROD + cf-space: tanf-prod + tf-path: ./terraform/production + + deploy-project-updates-site: + parameters: + cf-org: + default: CF_ORG + type: env_var_name + cf-space: + default: tanf-dev + type: string + cf-password: + type: env_var_name + default: CF_PASSWORD_DEV + cf-username: + type: env_var_name + default: CF_USERNAME_DEV + executor: docker-executor + working_directory: ~/tdp-deploy + steps: + - checkout + - sudo-check + - cf-check + - login-cloud-dot-gov: + cf-password: <> + cf-org: <> + cf-space: <> + cf-username: <> + - run: + name: Deploy TDP Project Updates Site + command: ./scripts/deploy-tdp-product-update-site.sh rolling tdp-project-updates diff --git a/.circleci/jobs.yml b/.circleci/jobs.yml deleted file mode 100644 index af93d9ef8..000000000 --- a/.circleci/jobs.yml +++ /dev/null @@ -1,282 +0,0 @@ -jobs: - secrets-check: - executor: docker-executor - steps: - - checkout - - run: - name: "git-secrets: Scan repository for committed secrets" - command: ./scripts/git-secrets-check.sh - - run: - name: "trufflehog: Scan repository for committed secrets" - command: ./scripts/trufflehog-check.sh $CIRCLE_BRANCH - - test-backend: - executor: machine-executor - steps: - - checkout - - docker-compose-check - - docker-compose-up-backend - - run: - name: Execute Python Linting Test - command: cd tdrs-backend; docker-compose run --rm web bash -c "flake8 ." - - run: - name: Run Unit Tests And Create Code Coverage Report - command: | - cd tdrs-backend; - docker-compose run --rm web bash -c "./wait_for_services.sh && pytest --cov-report=xml" - - upload-codecov: - component: backend - coverage-report: ./tdrs-backend/coverage.xml - - test-frontend: - executor: machine-executor - working_directory: ~/tdp-apps - steps: - - checkout - - install-nodejs-machine - - disable-npm-audit - - install-nodejs-packages: - app-dir: tdrs-frontend - - run: - name: Run ESLint - command: cd tdrs-frontend; npm run lint - - run: - name: Run Pa11y Accessibility Tests - command: cd tdrs-frontend; mkdir pa11y-screenshots/; npm run test:accessibility - - run: - name: Run Jest Unit Tests - command: cd tdrs-frontend; npm run test:ci - - upload-codecov: - component: frontend - coverage-report: ./tdrs-frontend/coverage/lcov.info - - store_artifacts: - path: tdrs-frontend/pa11y-screenshots/ - - backend-owasp-scan: - executor: machine-executor - working_directory: ~/tdp-apps - steps: - - checkout - - docker-compose-check - - docker-compose-up-backend - - run: - name: Wait for Django to become available - command: | - cd tdrs-backend; - docker-compose run --rm zaproxy bash -c \ - "PATH=$PATH:/home/zap/.local/bin && - pip install wait-for-it && - wait-for-it --service http://web:8080 \ - --timeout 60 \ - -- echo \"Django is ready\"" - - run-owasp-scan: - environment: circle - target: backend - - make_erd: - executor: machine-executor - working_directory: ~/tdp_apps - steps: - - checkout - - docker-compose-check - - run: - name: Run graph_models - command: | - cd tdrs-backend; - docker-compose run --rm web bash -c \ - "./manage.py graph_models -a -g -o tdp_erd.png" - - store_artifacts: - path: tdrs-backend/tdp_erd.png - - frontend-owasp-scan: - executor: machine-executor - working_directory: ~/tdp-apps - steps: - - checkout - - docker-compose-check - - docker-compose-up-frontend - - run: - name: Wait for frontend to become available - command: | - cd tdrs-frontend; - docker-compose run --rm zaproxy bash -c \ - "PATH=$PATH:/home/zap/.local/bin && - pip install wait-for-it && - wait-for-it --service http://tdp-frontend/ \ - --timeout 60 \ - -- echo \"Frontend is ready\"" - - run-owasp-scan: - environment: circle - target: frontend - - nightly-owasp-scan: - executor: machine-executor - working_directory: ~/tdp-apps - parameters: - cf_password: - type: string - default: CF_PASSWORD_STAGING - cf_username: - type: string - default: CF_USERNAME_STAGING - cf_space: - type: string - default: tanf-staging - target_env: - type: enum - enum: [ "staging", "develop", "prod" ] - steps: - - checkout - - sudo-check - - cf-check - - docker-compose-check - - run-owasp-scan: - environment: nightly - target: backend - target_env: <> - - run-owasp-scan: - environment: nightly - target: frontend - target_env: <> - - login-cloud-dot-gov: - cf-password: <> - cf-space: <> - cf-username: <> - - run: - name: Run post-processing task to record OWASP ZAP results - command: | - # Construct the project slug from the current branch name and user - PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME - - # These environment variables are exported to Circle CI's BASH_ENV - # by the zap-scanner.sh script for each respective app target. - CMD_ARGS=( - "$CIRCLE_BUILD_NUM" - --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} - --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} - --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} - --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} - --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} - --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} - --project-slug $PROJECT_SLUG - ) - # Evaluate the full command before passing it in so it doesn't - # get improperly interpolated by Cloud.gov. - CMD="python manage.py process_owasp_scan ${CMD_ARGS[@]}" - # Submit a CF Task for execution that will run the necessary command - cf run-task tdp-backend-<< parameters.target_env >> \ - --command "$CMD" \ - --name nightly-owasp-scan - deploy-infrastructure-dev: - executor: terraform/default - working_directory: ~/tdp-deploy - parameters: - target_env: - type: string - steps: - - deploy-infrastructure: - cf-app: << parameters.target_env >> - - deploy-dev: - executor: docker-executor - parameters: - target_env: - type: string - working_directory: ~/tdp-deploy - steps: - - deploy-cloud-dot-gov: - backend-appname: tdp-backend-<< parameters.target_env >> - frontend-appname: tdp-frontend-<< parameters.target_env >> - - deploy-infrastructure-staging: - executor: terraform/default - working_directory: ~/tdp-deploy - steps: - - deploy-infrastructure: - cf-password: CF_PASSWORD_STAGING - cf-username: CF_USERNAME_STAGING - cf-space: tanf-staging - tf-path: ./terraform/staging - - deploy-staging: - executor: docker-executor - working_directory: ~/tdp-deploy - steps: - - deploy-cloud-dot-gov: - backend-appname: tdp-backend-staging - frontend-appname: tdp-frontend-staging - cf-password: CF_PASSWORD_STAGING - cf-space: tanf-staging - cf-username: CF_USERNAME_STAGING - - deploy-infrastructure-develop: - executor: terraform/default - working_directory: ~/tdp-deploy - steps: - - deploy-infrastructure: - cf-password: CF_PASSWORD_STAGING - cf-username: CF_USERNAME_STAGING - cf-space: tanf-staging - tf-path: ./terraform/staging - - deploy-develop: - executor: docker-executor - working_directory: ~/tdp-deploy - steps: - - deploy-cloud-dot-gov: - backend-appname: tdp-backend-develop - frontend-appname: tdp-frontend-develop - cf-password: CF_PASSWORD_STAGING - cf-space: tanf-staging - cf-username: CF_USERNAME_STAGING - - deploy-project-updates-site: - parameters: - cf-org: - default: CF_ORG - type: env_var_name - cf-space: - default: tanf-dev - type: string - cf-password: - type: env_var_name - default: CF_PASSWORD_DEV - cf-username: - type: env_var_name - default: CF_USERNAME_DEV - executor: docker-executor - working_directory: ~/tdp-deploy - steps: - - checkout - - sudo-check - - cf-check - - login-cloud-dot-gov: - cf-password: <> - cf-org: <> - cf-space: <> - cf-username: <> - - run: - name: Deploy TDP Project Updates Site - command: ./scripts/deploy-tdp-product-update-site.sh rolling tdp-project-updates - - deploy-infrastructure-production: - executor: terraform/default - working_directory: ~/tdp-deploy - steps: - - deploy-infrastructure: - cf-password: CF_PASSWORD_PROD - cf-username: CF_USERNAME_PROD - cf-space: tanf-prod - tf-path: ./terraform/production - - deploy-production: - executor: docker-executor - working_directory: ~/tdp-deploy - steps: - - deploy-cloud-dot-gov: - environment: production - backend-appname: tdp-backend-prod - frontend-appname: tdp-frontend-prod - cf-password: CF_PASSWORD_PROD - cf-space: tanf-prod - cf-username: CF_USERNAME_PROD diff --git a/.circleci/owasp/commands.yml b/.circleci/owasp/commands.yml new file mode 100644 index 000000000..ae36aecba --- /dev/null +++ b/.circleci/owasp/commands.yml @@ -0,0 +1,31 @@ +# commands: + run-owasp-scan: + description: Runs OWASP ZAP scanner and stores resulting artifacts. + parameters: + environment: + description: The environment this script is being run for. + type: enum + enum: ["circle", "nightly"] + target: + description: The target application to be scanned. + type: enum + enum: ["backend", "frontend"] + target_env: + description: The target environment to be scanned. + type: enum + default: "develop" + enum: ["staging", "develop", "prod"] + steps: + - run: + name: Execute OWASP ZAP vulnerability scan + #wait up to 60min for scan + no_output_timeout: 60m + command: ./scripts/zap-scanner.sh <> <> <> + - store_artifacts: + path: tdrs-<>/reports/owasp_report.html + + docker-compose-up-frontend: + steps: + - run: + name: Build and spin-up React application + command: cd tdrs-frontend; docker-compose up -d --build diff --git a/.circleci/owasp/jobs.yml b/.circleci/owasp/jobs.yml new file mode 100644 index 000000000..f7348c005 --- /dev/null +++ b/.circleci/owasp/jobs.yml @@ -0,0 +1,101 @@ +# jobs: + backend-owasp-scan: + executor: machine-executor + working_directory: ~/tdp-apps + steps: + - checkout + - docker-compose-check + - docker-compose-up-backend + - run: + name: Wait for Django to become available + command: | + cd tdrs-backend; + docker-compose run --rm zaproxy bash -c \ + "PATH=$PATH:/home/zap/.local/bin && + pip install wait-for-it && + wait-for-it --service http://web:8080 \ + --timeout 60 \ + -- echo \"Django is ready\"" + - run-owasp-scan: + environment: circle + target: backend + + frontend-owasp-scan: + executor: machine-executor + working_directory: ~/tdp-apps + steps: + - checkout + - docker-compose-check + - docker-compose-up-frontend + - run: + name: Wait for frontend to become available + command: | + cd tdrs-frontend; + docker-compose run --rm zaproxy bash -c \ + "PATH=$PATH:/home/zap/.local/bin && + pip install wait-for-it && + wait-for-it --service http://tdp-frontend/ \ + --timeout 60 \ + -- echo \"Frontend is ready\"" + - run-owasp-scan: + environment: circle + target: frontend + + nightly-owasp-scan: + executor: machine-executor + working_directory: ~/tdp-apps + parameters: + cf_password: + type: string + default: CF_PASSWORD_STAGING + cf_username: + type: string + default: CF_USERNAME_STAGING + cf_space: + type: string + default: tanf-staging + target_env: + type: enum + enum: [ "staging", "develop", "prod" ] + steps: + - checkout + - sudo-check + - cf-check + - docker-compose-check + - run-owasp-scan: + environment: nightly + target: backend + target_env: <> + - run-owasp-scan: + environment: nightly + target: frontend + target_env: <> + - login-cloud-dot-gov: + cf-password: <> + cf-space: <> + cf-username: <> + - run: + name: Run post-processing task to record OWASP ZAP results + command: | + # Construct the project slug from the current branch name and user + PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME + + # These environment variables are exported to Circle CI's BASH_ENV + # by the zap-scanner.sh script for each respective app target. + CMD_ARGS=( + "$CIRCLE_BUILD_NUM" + --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0} + --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0} + --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0} + --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0} + --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0} + --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0} + --project-slug $PROJECT_SLUG + ) + # Evaluate the full command before passing it in so it doesn't + # get improperly interpolated by Cloud.gov. + CMD="python manage.py process_owasp_scan ${CMD_ARGS[@]}" + # Submit a CF Task for execution that will run the necessary command + cf run-task tdp-backend-<< parameters.target_env >> \ + --command "$CMD" \ + --name nightly-owasp-scan diff --git a/.circleci/util/commands.yml b/.circleci/util/commands.yml new file mode 100644 index 000000000..982cf545b --- /dev/null +++ b/.circleci/util/commands.yml @@ -0,0 +1,16 @@ +# commands: + cf-check: + steps: + - run: + name: Ensure cf cli is installed, otherwise install it. + command: sudo ./scripts/cf-check.sh + + # This allows us to use the orb stanza for node/install within other commands + # NOTE: This doesn't work correctly on machine executors + install-nodejs: node/install + + sudo-check: + steps: + - run: + name: Ensure sudo is installed, otherwise install it. + command: ./scripts/sudo-check.sh diff --git a/.circleci/util/jobs.yml b/.circleci/util/jobs.yml new file mode 100644 index 000000000..a994bf981 --- /dev/null +++ b/.circleci/util/jobs.yml @@ -0,0 +1,15 @@ +# jobs: + make_erd: + executor: machine-executor + working_directory: ~/tdp_apps + steps: + - checkout + - docker-compose-check + - run: + name: Run graph_models + command: | + cd tdrs-backend; + docker-compose run --rm web bash -c \ + "./manage.py graph_models -a -g -o tdp_erd.png" + - store_artifacts: + path: tdrs-backend/tdp_erd.png diff --git a/.circleci/workflows.yml b/.circleci/workflows.yml index b39046ac7..c87c9575a 100644 --- a/.circleci/workflows.yml +++ b/.circleci/workflows.yml @@ -24,53 +24,6 @@ workflows: requires: - deploy-infrastructure-dev - nightly: - jobs: - - nightly-owasp-scan: - target_env: develop - filters: - branches: - only: - - develop - - nightly-owasp-scan: - target_env: staging - filters: - branches: - only: - - main - - nightly-owasp-scan: - target_env: prod - cf_password: CF_PASSWORD_PROD - cf_username: CF_USERNAME_PROD - cf_space: tanf-prod - filters: - branches: - only: - - master - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - develop - - main - - master - - erd: - jobs: - - make_erd: - filters: - branches: - only: - develop - - owasp-scan: - when: << pipeline.parameters.run_owasp_scan >> - jobs: - - backend-owasp-scan - - frontend-owasp-scan - staging-deployment: unless: << pipeline.parameters.run_dev_deployment >> jobs: @@ -119,3 +72,51 @@ workflows: branches: only: - master + + owasp-scan: + when: << pipeline.parameters.run_owasp_scan >> + jobs: + - backend-owasp-scan + - frontend-owasp-scan + + nightly: + jobs: + - nightly-owasp-scan: + target_env: develop + filters: + branches: + only: + - develop + - nightly-owasp-scan: + target_env: staging + filters: + branches: + only: + - main + - nightly-owasp-scan: + target_env: prod + cf_password: CF_PASSWORD_PROD + cf_username: CF_USERNAME_PROD + cf_space: tanf-prod + filters: + branches: + only: + - master + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - develop + - main + - master + + erd: + jobs: + - make_erd: + filters: + branches: + only: + develop + From 6f8e8a9b1d606daa3e13919425b02d19abdea32c Mon Sep 17 00:00:00 2001 From: Cameron Smart Date: Mon, 21 Nov 2022 09:07:58 -0800 Subject: [PATCH 4/4] added separate workflow files --- .circleci/build-and-test/workflows.yml | 14 ++++++ .circleci/{ => deployment}/workflows.yml | 64 +----------------------- .circleci/generate_config.sh | 6 ++- .circleci/owasp/workflows.yml | 39 +++++++++++++++ .circleci/util/workflows.yml | 8 +++ 5 files changed, 67 insertions(+), 64 deletions(-) create mode 100644 .circleci/build-and-test/workflows.yml rename .circleci/{ => deployment}/workflows.yml (53%) create mode 100644 .circleci/owasp/workflows.yml create mode 100644 .circleci/util/workflows.yml diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml new file mode 100644 index 000000000..97b4f9bb8 --- /dev/null +++ b/.circleci/build-and-test/workflows.yml @@ -0,0 +1,14 @@ +# workflows: + build-and-test: + unless: + or: + - << pipeline.parameters.run_dev_deployment >> + - << pipeline.parameters.run_owasp_scan >> + jobs: + - secrets-check + - test-frontend: + requires: + - secrets-check + - test-backend: + requires: + - secrets-check diff --git a/.circleci/workflows.yml b/.circleci/deployment/workflows.yml similarity index 53% rename from .circleci/workflows.yml rename to .circleci/deployment/workflows.yml index c87c9575a..f8cfd2590 100644 --- a/.circleci/workflows.yml +++ b/.circleci/deployment/workflows.yml @@ -1,18 +1,4 @@ -workflows: - build-and-test: - unless: - or: - - << pipeline.parameters.run_dev_deployment >> - - << pipeline.parameters.run_owasp_scan >> - jobs: - - secrets-check - - test-frontend: - requires: - - secrets-check - - test-backend: - requires: - - secrets-check - +# workflows: dev-deployment: when: << pipeline.parameters.run_dev_deployment >> @@ -72,51 +58,3 @@ workflows: branches: only: - master - - owasp-scan: - when: << pipeline.parameters.run_owasp_scan >> - jobs: - - backend-owasp-scan - - frontend-owasp-scan - - nightly: - jobs: - - nightly-owasp-scan: - target_env: develop - filters: - branches: - only: - - develop - - nightly-owasp-scan: - target_env: staging - filters: - branches: - only: - - main - - nightly-owasp-scan: - target_env: prod - cf_password: CF_PASSWORD_PROD - cf_username: CF_USERNAME_PROD - cf_space: tanf-prod - filters: - branches: - only: - - master - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - develop - - main - - master - - erd: - jobs: - - make_erd: - filters: - branches: - only: - develop - diff --git a/.circleci/generate_config.sh b/.circleci/generate_config.sh index 4caf1a80e..696d3fdf1 100644 --- a/.circleci/generate_config.sh +++ b/.circleci/generate_config.sh @@ -19,6 +19,10 @@ cat owasp/jobs.yml >> generated_config.yml cat util/jobs.yml >> generated_config.yml # Workflows -cat workflows.yml >> generated_config.yml +echo 'workflows:' >> generated_config.yml +cat build-and-test/workflows.yml >> generated_config.yml +cat deployment/workflows.yml >> generated_config.yml +cat owasp/workflows.yml >> generated_config.yml +cat util/workflows.yml >> generated_config.yml cat generated_config.yml \ No newline at end of file diff --git a/.circleci/owasp/workflows.yml b/.circleci/owasp/workflows.yml new file mode 100644 index 000000000..1b5cc90b1 --- /dev/null +++ b/.circleci/owasp/workflows.yml @@ -0,0 +1,39 @@ +# workflows: + owasp-scan: + when: << pipeline.parameters.run_owasp_scan >> + jobs: + - backend-owasp-scan + - frontend-owasp-scan + + nightly: + jobs: + - nightly-owasp-scan: + target_env: develop + filters: + branches: + only: + - develop + - nightly-owasp-scan: + target_env: staging + filters: + branches: + only: + - main + - nightly-owasp-scan: + target_env: prod + cf_password: CF_PASSWORD_PROD + cf_username: CF_USERNAME_PROD + cf_space: tanf-prod + filters: + branches: + only: + - master + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - develop + - main + - master diff --git a/.circleci/util/workflows.yml b/.circleci/util/workflows.yml new file mode 100644 index 000000000..6a9422c31 --- /dev/null +++ b/.circleci/util/workflows.yml @@ -0,0 +1,8 @@ +# workflows: + erd: + jobs: + - make_erd: + filters: + branches: + only: + develop