forked from GoogleCloudPlatform/bigquery-utils
-
Notifications
You must be signed in to change notification settings - Fork 0
/
cloudbuild.yaml
160 lines (158 loc) · 4.98 KB
/
cloudbuild.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
# Google Cloud Build script for bqutils
#
# This build script is used to unit test the BigQuery UDFs for every
# change pushed to the udfs/ directory.
#
# Manual Execution:
# Use the below command to invoke the build manually. Note the substitutions for
# BRANCH_NAME and REVISION_ID. These variables are normally populated when the
# build is executed via build triggers but will be empty during manual
# execution. Dummy branch and revisions can be passed during manual execution so
# the artifacts can be uploaded upon build completion.
#
# gcloud builds submit . --config=cloudbuild.yaml
#
steps:
############################################################
# Dynamically create the package.json file based off the libs
# specified in the js_libs/js_libs.yaml file.
############################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci
id: generate_js_libs_package_json
entrypoint: python3
args:
- tests/udf_test_utils.py
- --generate-js-libs-package-json
###########################################################
# Install npm packages based off the package.json file
# created in the previous step.
###########################################################
- name: node
id: install_npm_packages
entrypoint: npm
args:
- install
############################################################
# Dynamically create webpack config files needed by webpack
# to build npm packages into single .js files which will be
# hosted on GCS and used by BigQuery UDFs.
############################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci
id: generate_webpack_configs
entrypoint: python3
args:
- tests/udf_test_utils.py
- --generate-webpack-configs
waitFor:
- install_npm_packages
- generate_js_libs_package_json
###########################################################
# Build (via webpack) all js libraries for BigQuery UDFs
###########################################################
- name: node
id: build_bq_js_libs
entrypoint: npm
args:
- run-script
- build-all-libs
waitFor:
- generate_webpack_configs
- install_npm_packages
###########################################################
# Copy all libs to GCS bucket
###########################################################
- name: gcr.io/google.com/cloudsdktool/cloud-sdk
id: copy_js_to_gcs
entrypoint: gsutil
args:
- '-m'
- cp
- js_builds/*
- ${_JS_BUCKET}
waitFor:
- build_bq_js_libs
###########################################################
# Create BigQuery datasets specifically for testing
###########################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci
id: create_test_datasets
entrypoint: python3
args:
- tests/udf_test_utils.py
- --create-test-datasets
env:
- SHORT_SHA=${SHORT_SHA}
- BQ_PROJECT_ID=${PROJECT_ID}
###########################################################
# Create UDF signatures with empty bodies to prevent
# UDF creation errors in next step when one UDF depends
# on another UDF.
###########################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci
id: create_udf_signatures
entrypoint: pytest
args:
- --workers
- auto # 1 worker per core
- --tests-per-worker
- auto # Up to 50 tests per worker
- tests/create_udf_signatures.py
env:
- SHORT_SHA=${SHORT_SHA}
- BQ_PROJECT_ID=${PROJECT_ID}
waitFor:
- copy_js_to_gcs
- create_test_datasets
###########################################################
# Create UDFs in the test datasets
###########################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci
id: create_udfs
entrypoint: pytest
args:
- --workers
- auto # 1 worker per core
- --tests-per-worker
- auto # Up to 50 tests per worker
- tests/test_create_udfs.py
env:
- SHORT_SHA=${SHORT_SHA}
- BRANCH_NAME=${BRANCH_NAME}
- BQ_PROJECT_ID=${PROJECT_ID}
waitFor:
- create_udf_signatures
###########################################################
# Run UDF Unit Tests
###########################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci
id: run_udf_unit_tests
entrypoint: pytest
args:
- --workers
- auto # 1 worker per core
- --tests-per-worker
- auto # Up to 50 tests per worker
- tests/test_run_udfs.py
env:
- SHORT_SHA=${SHORT_SHA}
- BRANCH_NAME=${BRANCH_NAME}
- BQ_PROJECT_ID=${PROJECT_ID}
waitFor:
- create_udfs
###########################################################
# Delete the test datasets that were created
###########################################################
- name: gcr.io/$PROJECT_ID/bq_udf_ci
id: delete_test_datasets
entrypoint: python3
args:
- tests/udf_test_utils.py
- --delete-test-datasets
env:
- SHORT_SHA=${SHORT_SHA}
- BQ_PROJECT_ID=${PROJECT_ID}
waitFor:
- run_udf_unit_tests
timeout: 1800s # 30 minutes
options:
machineType: N1_HIGHCPU_32