forked from GoogleCloudPlatform/bigquery-utils
-
Notifications
You must be signed in to change notification settings - Fork 0
/
cloudbuild_deploy.yaml
102 lines (101 loc) · 3.27 KB
/
cloudbuild_deploy.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
# Google Cloud Build script for bqutils
#
# This build script is used to build and deploy the BigQuery UDFs.
#
# Manual Execution:
# Use the below command to invoke the build manually. Note the substitutions for
# BRANCH_NAME and REVISION_ID. These variables are normally populated when the
# build is executed via build triggers but will be empty during manual
# execution. Dummy branch and revisions can be passed during manual execution so
# the artifacts can be uploaded upon build completion.
#
# gcloud builds submit . --config=cloudbuild_deploy.yaml
#
steps:
############################################################
# Dynamically create the package.json file based off the libs
# specified in the js_libs/js_libs.yaml file.
############################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci
id: generate_js_libs_package_json
entrypoint: python3
args:
- tests/udf_test_utils.py
- --generate-js-libs-package-json
###########################################################
# Install npm packages based off the package.json file
# created in the previous step.
###########################################################
- name: node
id: install_npm_packages
entrypoint: npm
args:
- install
############################################################
# Dynamically create webpack config files needed by webpack
# to build npm packages into single .js files which will be
# hosted on GCS and used by BigQuery UDFs.
############################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci
id: generate_webpack_configs
entrypoint: python3
args:
- tests/udf_test_utils.py
- --generate-webpack-configs
waitFor:
- install_npm_packages
- generate_js_libs_package_json
###########################################################
# Build (via webpack) all js libraries for BigQuery UDFs
###########################################################
- name: node
id: build_bq_js_libs
entrypoint: npm
args:
- run-script
- build-all-libs
waitFor:
- generate_webpack_configs
- install_npm_packages
###########################################################
# Copy all libs to GCS bucket
###########################################################
- name: gcr.io/google.com/cloudsdktool/cloud-sdk
id: copy_js_to_gcs
entrypoint: gsutil
args:
- '-m'
- cp
- js_builds/*
- ${_JS_BUCKET}
waitFor:
- build_bq_js_libs
###########################################################
# Create BigQuery prod datasets for hosting UDFs
###########################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci
id: create_prod_datasets
entrypoint: python3
args:
- tests/udf_test_utils.py
- --create-prod-datasets
env:
- BQ_PROJECT_ID=${PROJECT_ID}
###########################################################
# Create UDFs in the prod datasets
###########################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci
id: create_prod_udfs
entrypoint: pytest
args:
- --workers
- '100'
- tests/test_create_udfs.py
env:
- BRANCH_NAME=${BRANCH_NAME}
- BQ_PROJECT_ID=${PROJECT_ID}
waitFor:
- copy_js_to_gcs
- create_prod_datasets
options:
machineType: N1_HIGHCPU_8