Artifacts bundle (#7)

flyteorg · Sep 26, 2023 · de6cd33 · de6cd33
1 parent cd0624c
commit de6cd33
Show file tree

Hide file tree

Showing 16 changed files with 4,151 additions and 16 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -7,10 +7,22 @@ FROM --platform=${BUILDPLATFORM} golang:1.19.1-bullseye AS flytebuilder
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"
 ENV GOOS linux
+ENV GOPRIVATE github.com/unionai/*
+RUN mkdir -m 0700 ~/.ssh
+RUN ssh-keyscan github.com >> ~/.ssh/known_hosts
+
 
 WORKDIR /flyteorg/build
-COPY go.mod go.sum ./
+COPY go.mod.main ./go.mod
+COPY go.sum.main ./go.sum
 RUN go mod download
+RUN echo '\
+[url "ssh://[email protected]/"]\n\
+  insteadOf = https://github.com/'\
+>> /root/.gitconfig
+
+COPY go.mod go.sum ./
+RUN --mount=type=ssh go mod download
 COPY cmd cmd
 COPY --from=flyteconsole /app/ cmd/single/dist
 RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/go/pkg/mod \

diff --git a/charts/flyte-sandbox/Chart.lock b/charts/flyte-sandbox/Chart.lock
@@ -14,5 +14,8 @@ dependencies:
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
   version: 12.1.9
-digest: sha256:e7155e540bbdb98f690eb12e2bd301a19d8b36833336f6991410cb44d8d9bb5e
-generated: "2023-03-31T09:25:07.80904-07:00"
+- name: redis
+  repository: https://charts.bitnami.com/bitnami
+  version: 17.15.6
+digest: sha256:1bcc5f102768a19c19d444b3460dabd0f90847b2d4423134f0ce9c7aa0a256ea
+generated: "2023-08-31T16:46:00.478623-07:00"
diff --git a/charts/flyte-sandbox/Chart.yaml b/charts/flyte-sandbox/Chart.yaml
@@ -44,3 +44,7 @@ dependencies:
     version: 12.1.9
     repository: https://charts.bitnami.com/bitnami
     condition: postgresql.enabled
+  - name: redis
+    version: 17.15.6
+    repository: https://charts.bitnami.com/bitnami
+    condition: redis.enabled
diff --git a/charts/flyte-sandbox/README.md b/charts/flyte-sandbox/README.md
@@ -11,6 +11,7 @@ A Helm chart for the Flyte local sandbox
 | file://../flyte-binary | flyte-binary | v0.1.10 |
 | https://charts.bitnami.com/bitnami | minio | 12.1.1 |
 | https://charts.bitnami.com/bitnami | postgresql | 12.1.9 |
+| https://charts.bitnami.com/bitnami | redis | 18.0.1 |
 | https://helm.twun.io/ | docker-registry | 2.2.2 |
 | https://kubernetes.github.io/dashboard/ | kubernetes-dashboard | 6.0.0 |
 
@@ -92,6 +93,13 @@ A Helm chart for the Flyte local sandbox
 | postgresql.volumePermissions.enabled | bool | `true` |  |
 | postgresql.volumePermissions.image.pullPolicy | string | `"Never"` |  |
 | postgresql.volumePermissions.image.tag | string | `"sandbox"` |  |
+| redis.auth.enabled | bool | `false` |  |
+| redis.enabled | bool | `true` |  |
+| redis.image.pullPolicy | string | `"Never"` |  |
+| redis.image.tag | string | `"sandbox"` |  |
+| redis.master.service.nodePorts.redis | int | `30004` |  |
+| redis.master.service.type | string | `"NodePort"` |  |
+| redis.replica.replicaCount | int | `0` |  |
 | sandbox.buildkit.enabled | bool | `true` |  |
 | sandbox.buildkit.image.pullPolicy | string | `"Never"` |  |
 | sandbox.buildkit.image.repository | string | `"moby/buildkit"` |  |

diff --git a/charts/flyte-sandbox/templates/artifacts/deployment.yaml b/charts/flyte-sandbox/templates/artifacts/deployment.yaml
@@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: artifact-service
+  name: artifact-service
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: artifact-service
+  template:
+    metadata:
+      labels:
+        app: artifact-service
+    spec:
+      containers:
+        - name: main
+          image: ghcr.io/unionai/artifacts:sandbox
+          env:
+            - name: DATABASE_URL
+              value: postgresql://postgres:[email protected]:5432/postgres
+            - name: REDIS_HOST
+              value: flyte-sandbox-redis-headless.flyte.svc.cluster.local
+            - name: REDIS_PORT
+              value: "6379"
+          ports:
+            - name: grpc
+              containerPort: 50051
+          livenessProbe:
+            initialDelaySeconds: 30
+            tcpSocket:
+              port: grpc
+          readinessProbe:
+            tcpSocket:
+              port: grpc
diff --git a/charts/flyte-sandbox/templates/artifacts/service.yaml b/charts/flyte-sandbox/templates/artifacts/service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: artifact-service
+  labels:
+    app: artifact-service
+spec:
+  ports:
+    - name: grpc
+      port: 50051
+      targetPort: 50051
+  selector:
+    app: artifact-service
+  type: ClusterIP
diff --git a/charts/flyte-sandbox/templates/proxy/configmap.yaml b/charts/flyte-sandbox/templates/proxy/configmap.yaml
@@ -108,6 +108,10 @@ data:
                                                   prefix: "/flyteidl.service.SignalService"
                                               route:
                                                   cluster: flyte_grpc
+                                            - match:
+                                                  prefix: "/flyteidl.artifact.ArtifactRegistry"
+                                              route:
+                                                  cluster: artifact
                                             {{- end }}
                                             {{- if index .Values "kubernetes-dashboard" "enabled" }}
                                             - match:
@@ -203,5 +207,19 @@ data:
                                           address: {{ .Release.Name }}-minio
                                           port_value: 9001
             {{- end }}
+            - name: artifact
+              connect_timeout: 0.25s
+              type: STRICT_DNS
+              lb_policy: ROUND_ROBIN
+              http2_protocol_options: {}
+              load_assignment:
+                  cluster_name: artifact
+                  endpoints:
+                      - lb_endpoints:
+                            - endpoint:
+                                  address:
+                                      socket_address:
+                                          address: artifact-service
+                                          port_value: 50051
 
 {{- end }}
diff --git a/charts/flyte-sandbox/values.yaml b/charts/flyte-sandbox/values.yaml
@@ -29,13 +29,23 @@ flyte-binary:
           accessKey: minio
           secretKey: miniostorage
     logging:
-      level: 6
+      level: 5
       plugins:
         kubernetes:
           enabled: true
           templateUri: |-
             http://localhost:30080/kubernetes-dashboard/#/log/{{.namespace }}/{{ .podName }}/pod?namespace={{ .namespace }}
     inline:
+      cloudEvents:
+        enable: true
+        transformToCloudEvents: true
+        type: redis
+        redis:
+          addr: "flyte-sandbox-redis-headless.flyte.svc.cluster.local:6379"
+      artifacts:
+        host: artifact-service.flyte.svc.cluster.local
+        port: 50051
+        insecure: true
       storage:
         signedURL:
           stowConfigOverride:
@@ -138,6 +148,21 @@ postgresql:
       tag: sandbox
       pullPolicy: Never
 
+redis:
+  enabled: true
+  image:
+    tag: sandbox
+    pullPolicy: Never
+  auth:
+    enabled: false
+  master:
+    service:
+      type: NodePort
+      nodePorts:
+        redis: 30004
+  replica:
+    replicaCount: 0
+
 sandbox:
   # dev Routes requests to an instance of Flyte running locally on a developer's
   # development environment. This is only usable if the flyte-binary chart is disabled.

diff --git a/docker/sandbox-bundled/Makefile b/docker/sandbox-bundled/Makefile
@@ -1,7 +1,7 @@
 define FLYTE_BINARY_BUILD
 mkdir -p images/tar/$(1)
 
-docker buildx build \
+docker buildx build --ssh default \
 	--build-arg FLYTECONSOLE_VERSION=$(FLYTECONSOLE_VERSION) \
 	--platform linux/$(1) \
 	--tag flyte-binary:sandbox \
@@ -29,13 +29,13 @@ manifests:
 		kustomize/dev > manifests/dev.yaml
 
 .PHONY: build
-build: flyte manifests
+build: manifests
 	[ -n "$(shell docker buildx ls | awk '/^flyte-sandbox / {print $$1}')" ] || \
 		docker buildx create --name flyte-sandbox \
 		--driver docker-container --driver-opt image=moby/buildkit:master \
 		--buildkitd-flags '--allow-insecure-entitlement security.insecure' \
 		--platform linux/arm64,linux/amd64
-	docker buildx build --builder flyte-sandbox --allow security.insecure --load \
+	docker buildx build --ssh default --builder flyte-sandbox --allow security.insecure --load \
 		--tag flyte-sandbox:latest .
 
 # Port map

diff --git a/docker/sandbox-bundled/images/manifest.txt b/docker/sandbox-bundled/images/manifest.txt
@@ -1,6 +1,7 @@
 docker.io/bitnami/bitnami-shell:sandbox=bitnami/bitnami-shell:11-debian-11-r76
 docker.io/bitnami/minio:sandbox=bitnami/minio:2023.1.25-debian-11-r0
 docker.io/bitnami/postgresql:sandbox=bitnami/postgresql:15.1.0-debian-11-r20
+docker.io/bitnami/redis:sandbox=bitnami/redis:7.2.0-debian-11-r3
 docker.io/envoyproxy/envoy:sandbox=envoyproxy/envoy:v1.23-latest
 docker.io/kubernetesui/dashboard:sandbox=kubernetesui/dashboard:v2.7.0
 docker.io/library/registry:sandbox=registry:2.8.1