cloudbuild.yaml

# Copyright 2019 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
steps:
- name: 'gcr.io/cloud-builders/git'
  entrypoint: 'bash'
  args: [
    './helpers/init_git_repo.sh'
  ]
  env: [
    'COMMIT_SHA=${COMMIT_SHA}',
    'BASE_REPO_URL=https://github.com/jaketf/ci-cd-for-data-processing-workflow.git',
    'BASE_BRANCH=${_BASE_BRANCH}',
    'PR_NUMBER=${_PR_NUMBER}'
  ]
  id: 'init-git-repo'
# Merge Master because this is a post-commit
- name: 'gcr.io/cloud-builders/git'
  args: ['merge', 'origin/master']
  waitFor: ['init-git-repo']
  id: 'merge-master'
# Run linters and relevant cloudbuilds (we have to do this again in case merge
# broke a unit test or static check)
# Individual cloudbuild.yaml's  (e.g. for a dataflow pipeline) are responsible
# for pushing their own artifacts to the appropriate gcs location in the
# artifacts project and updating the necessary references in composer.
- name: 'google/cloud-sdk'
  waitFor: ['merge-master']
  entrypoint: 'bash'
  args: [
    './helpers/run_relevant_cloudbuilds.sh',
    'cloudbuild.yaml',
    '--substitutions=_SHORT_SHA=${SHORT_SHA},_COMPOSER_REGION=${_COMPOSER_REGION},_COMPOSER_ENV_NAME=${_COMPOSER_ENV_NAME},_DATAFLOW_JAR_BUCKET=${_DATAFLOW_JAR_BUCKET}'
  ]
  id: 'run-builds'
# [BigQuery]
# Copy SQL to DAGs folder.
- name: 'google/cloud-sdk'
  waitFor: ['run-builds']
  entrypoint: 'gsutil'
  args: [
          'rsync','-r', '-d',
          'sql', '${_COMPOSER_DAG_BUCKET}dags/sql'
        ]
  dir: './bigquery/'
  id: 'deploy-sql-queries-for-composer'
# [Composer]
# Render AirflowVariables.json
- name: 'gcr.io/${PROJECT_ID}/envsubst'
  waitFor: ['-']
  env: [
          "GCP_PROJECT_ID=${PROJECT_ID}",
          "COMPOSER_REGION=${_COMPOSER_REGION}",
          "DATAFLOW_JAR_BUCKET=${_DATAFLOW_ARTIFACTS_BUCKET}",
          "INPUT_BUCKET=${_WORDCOUNT_INPUT_BUCKET}",
          "REF_BUCKET=${_WORDCOUNT_REF_BUCKET}",
          "RESULT_BUCKET=${_WORDCOUNT_RESULT_BUCKET}",
          "DATAFLOW_STAGING_BUCKET=${_DATAFLOW_STAGING_BUCKET}",
       ] 
  args: ['AirflowVariables.json']
  dir: './composer/config'
  id: 'render-airflow-variables'
# Add .airflowignore to GCS DAGs folder.
- name: 'google/cloud-sdk'
  waitFor: ['run-builds']
  entrypoint: 'gcloud'
  args: [
          'composer', 'environments', 'storage', 'dags', 'import',
          '--source','.airflowignore',
          '--environment', '${_COMPOSER_ENV_NAME}',
          '--location', '${_COMPOSER_REGION}'
        ]
  dir: './composer/dags/'
  id: 'deploy-airflowignore'
# Stage files for running the example.
- name: 'google/cloud-sdk'
  waitFor: ['-']
  entrypoint: 'gsutil'
  args: [
          'cp',
          'support-files/input.txt',
          'gs://${_WORDCOUNT_INPUT_BUCKET}'
        ]
  dir: './composer/dags'
  id: 'deploy-test-input-file'
- name: 'google/cloud-sdk'
  waitFor: ['-']
  entrypoint: 'gsutil'
  args: [
          'cp',
          'support-files/ref.txt',
          'gs://${_WORDCOUNT_REF_BUCKET}'
        ]
  dir: './composer/dags'
  id: 'deploy-test-ref-file'
# Stage AirflowVariables.json to data directory to be synced to workers.
- name: 'google/cloud-sdk'
  waitFor: ['render-airflow-variables']
  entrypoint: 'gcloud'
  args: [
          'composer', 'environments', 'storage', 'data', 'import',
          '--location=${_COMPOSER_REGION}',
          '--environment=${_COMPOSER_ENV_NAME}',
          '--source','AirflowVariables.json',
          '--destination', 'config'
        ]
  dir: './composer/config/'
  id: 'stage-airflow-variables'
# Import AirflowVariables.json 
- name: 'google/cloud-sdk'
  waitFor: ['stage-airflow-variables']
  entrypoint: 'gcloud'
  args: [
          'composer', 'environments', 'run', 
          '--location=${_COMPOSER_REGION}',
          '${_COMPOSER_ENV_NAME}',
          'variables', '--',
          '--import', '/home/airflow/gcs/data/config/AirflowVariables.json'
        ] 
  id: 'import-airflow-variables'
# Sync plugins to GCS plugins dir
- name: 'google/cloud-sdk'
  waitFor: ['run-builds']
  entrypoint: 'gsutil'
  args: [
          'rsync','-r', '-d',
          'plugins/',
          '${_COMPOSER_DAG_BUCKET}plugins'
        ]
  dir: './composer/'
  id: 'deploy-custom-plugins'
# Sync DAGs to data dir for integration test parsing in target Composer Environment.
- name: 'google/cloud-sdk'
  waitFor: ['deploy-custom-plugins']
  entrypoint: 'gsutil'
  args: [
          'rsync','-r', '-d',
          'dags/',
          '${_COMPOSER_DAG_BUCKET}data/test-dags/$BUILD_ID'
        ]
  dir: './composer/'
  id: 'stage-for-integration-test'
# Run integration tests parsing in target Composer Environment.
- name: 'google/cloud-sdk'
  waitFor: ['stage-for-integration-test']
  entrypoint: 'gcloud'
  args: [
          'composer', 'environments', 'run', 
          '--location', '${_COMPOSER_REGION}',
          '${_COMPOSER_ENV_NAME}', 
          'list_dags', '--', 
          '-sd', '/home/airflow/gcs/data/test-dags/$BUILD_ID'
        ] 
  id: 'dag-parse-integration-test'
# Clean up. 
- name: 'google/cloud-sdk'
  waitFor: ['dag-parse-integration-test']
  entrypoint: 'gsutil'
  args: [
          '-m', 'rm','-r',
          '${_COMPOSER_DAG_BUCKET}data/test-dags/$BUILD_ID'
        ]
  dir: './composer/'
  id: 'clean-up-data-dir-dags'
# pull dags deployer golang app.
- name: gcr.io/cloud-builders/docker
  waitFor: ['-']
  entrypoint: 'bash'
  args: [
          '-c',
          'docker pull gcr.io/${_ARTIFACTS_PROJECT_ID}/deploydags:latest || exit 0'
        ]
  id: 'pull-deploydags'
# build with cache
- name: gcr.io/cloud-builders/docker
  waitFor: ['pull-deploydags']
  dir: './composer/cloudbuild/go/dagsdeployer'
  args: [
          'build', 
          '-t', 'gcr.io/${PROJECT_ID}/deploydags:latest',
          '--cache-from', 'gcr.io/${_ARTIFACTS_PROJECT_ID}/deploydags:latest',
          '.'
        ]
  id: 'build-deploydags'
# Run dags deployer golang app. 
- name: gcr.io/${PROJECT_ID}/deploydags
  dir: './composer'
  waitFor: [
          'run-builds',
          'build-deploydags',
          'clean-up-data-dir-dags',
          ]
  args: [
          '-dagList=./config/running_dags.txt',
          '-dagsFolder=./dags',
          '-project=${PROJECT_ID}',
          '-region=${_COMPOSER_REGION}',
          '-composerEnv=${_COMPOSER_ENV_NAME}',
          '-dagBucketPrefix=${_COMPOSER_DAG_BUCKET}dags',
          '-replace'
        ]
  id: 'run-deploydags'
artifacts:
  images: ['gcr.io/${_ARTIFACTS_PROJECT_ID}/deploydags']
options:
  machineType: 'N1_HIGHCPU_32'  # For test and deploy dags parallelization.