-
Notifications
You must be signed in to change notification settings - Fork 11
167 lines (152 loc) · 7.06 KB
/
dagster-cloud-deploy.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
name: Dagster Cloud Hybrid Deployment
on:
push: # For full deployments
branches:
- "main"
- "master"
pull_request: # For branch deployments
types: [opened, synchronize, reopened, closed]
concurrency:
# Cancel in-progress deploys to the same branch
group: ${{ github.ref }}
cancel-in-progress: true
env:
# The organization name in Dagster Cloud
DAGSTER_CLOUD_ORGANIZATION: "<organization-name>"
# The API token from https://dagster.cloud/ should be stored in Secrets
DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
# Path to the root folder containing the dagster project
DAGSTER_PROJECT_DIR: "."
# Path to dagster_cloud.yaml relative to DAGSTER_PROJECT_DIR
DAGSTER_CLOUD_YAML_PATH: "dagster_cloud.yaml"
# The IMAGE_REGISTRY should match the 'registry:'' in dagster_cloud.yaml
IMAGE_REGISTRY: "<account-id>.dkr.ecr.us-west-2.amazonaws.com/<image-name>"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
jobs:
dagster-cloud-deploy:
runs-on: ubuntu-20.04
steps:
# If this is a closed PR the prerun step closes the branch deployment and returns
# output.result='skip' which is used to skip other steps in this workflow.
- name: Pre-run checks
id: prerun
uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
# Checkout the project
- name: Checkout
uses: actions/checkout@v4
if: steps.prerun.outputs.result != 'skip'
with:
ref: ${{ github.head_ref }}
# Validate dagster_cloud.yaml and the connection to dagster.cloud
- name: Validate configuration
id: ci-validate
if: steps.prerun.outputs.result != 'skip'
uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
with:
command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_YAML_PATH }}"
# Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session
- name: Initialize build session
id: ci-init
if: steps.prerun.outputs.result != 'skip'
uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
with:
project_dir: ${{ env.DAGSTER_PROJECT_DIR }}
dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_YAML_PATH }}
# The full deployment name. If this run is for a PR, this value is ignored and a branch
# deployment is used.
deployment: 'prod'
# Any value can be used as the docker image tag. It is recommended to use a unique value
# for each build so that multiple builds do not overwrite each other.
- name: Generate docker image tag
id: generate-image-tag
if: steps.prerun.outputs.result != 'skip'
run: echo "IMAGE_TAG=$GITHUB_SHA-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" >> $GITHUB_ENV && echo $IMAGE_TAG
# Enable buildx for caching
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
# Building and deploying the docker image requires a login step specific to the container
# registry.
# Multiple examples are provided below.
# # AWS ECR
# # https://github.com/aws-actions/amazon-ecr-login
# - name: Configure AWS credentials
# if: steps.prerun.outputs.result != 'skip'
# uses: aws-actions/configure-aws-credentials@v4
# with:
# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# aws-region: ${{ secrets.AWS_REGION }}
# - name: Login to ECR
# if: steps.prerun.outputs.result != 'skip'
# uses: aws-actions/amazon-ecr-login@v2
# with:
# mask-password: 'true'
# # DockerHub
# # https://github.com/docker/login-action#docker-hub
# - name: Login to Docker Hub
# if: steps.prerun.outputs.result != 'skip'
# uses: docker/login-action@v1
# with:
# username: ${{ secrets.DOCKERHUB_USERNAME }}
# password: ${{ secrets.DOCKERHUB_TOKEN }}
# # GitHub Container Registry
# # https://github.com/docker/login-action#github-container-registry
# - name: Login to GitHub Container Registry
# if: steps.prerun.outputs.result != 'skip'
# uses: docker/login-action@v1
# with:
# registry: ghcr.io
# username: ${{ github.actor }}
# password: ${{ secrets.GITHUB_TOKEN }}
# # GCR
# # https://github.com/docker/login-action#google-container-registry-gcr
# - name: Login to GCR
# if: steps.prerun.outputs.result != 'skip'
# uses: docker/login-action@v1
# with:
# registry: gcr.io
# username: _json_key
# password: ${{ secrets.GCR_JSON_KEY }}
# Build the "quickstart_etl" location.
# For each code location, the "build-push-action" builds the docker
# image and a "set-build-output" command records the image tag for each code location.
# To re-use the same docker image across multiple code locations, build the docker image once
# and specify the same tag in multiple "set-build-output" commands. To use a different docker
# image for each code location, use multiple "build-push-actions" with a location specific
# tag.
- name: Build and upload Docker image for "quickstart_etl"
if: steps.prerun.outputs.result != 'skip'
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: ${{ env.IMAGE_REGISTRY }}:${{ env.IMAGE_TAG }}-quickstart_etl
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Update build session with image tag for quickstart_etl
id: ci-set-build-output-example-location
if: steps.prerun.outputs.result != 'skip'
uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
with:
command: "ci set-build-output --location-name=quickstart_etl --image-tag=$IMAGE_TAG-quickstart_etl"
# Deploy all code locations in this build session to Dagster Cloud
- name: Deploy to Dagster Cloud
id: ci-deploy
if: steps.prerun.outputs.result != 'skip'
uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
with:
command: "ci deploy"
# Update a PR comment - this runs always() so the comment is updated on success and failure
- name: Update PR comment for branch deployments
id: ci-notify
if: steps.prerun.outputs.result != 'skip' && always()
uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
with:
command: "ci notify --project-dir=${{ env.DAGSTER_PROJECT_DIR }}"
# Generate a summary that shows up on the Workflow Summary page
- name: Generate a summary
id: ci-summary
if: steps.prerun.outputs.result != 'skip' && always()
uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
with:
command: "ci status --output-format=markdown >> $GITHUB_STEP_SUMMARY"