setup in-cluster image registry

Signed-off-by: akihikokuroda <[email protected]>

Makefile

@@ -73,7 +73,7 @@ verify: tidy fmt generate ## Verify the current code generation and lint
 ###########
 # Testing #
 ###########
-.PHONY: test test-unit test-e2e
+.PHONY: test test-unit test-e2e image-registry
 
 ##@ testing:
 
@@ -89,7 +89,7 @@ test-e2e: ginkgo ## Run the e2e tests
 	$(GINKGO) -trace -progress $(FOCUS) test/e2e
 
 e2e: KIND_CLUSTER_NAME=rukpak-e2e
-e2e: run kind-load-bundles test-e2e kind-cluster-cleanup ## Run e2e tests against an ephemeral kind cluster
+e2e: run image-registry kind-load-bundles test-e2e kind-cluster-cleanup ## Run e2e tests against an ephemeral kind cluster
 
 kind-cluster: kind kind-cluster-cleanup ## Standup a kind cluster
 	$(KIND) create cluster --name ${KIND_CLUSTER_NAME}
@@ -98,6 +98,9 @@ kind-cluster: kind kind-cluster-cleanup ## Standup a kind cluster
 kind-cluster-cleanup: kind ## Delete the kind cluster
 	$(KIND) delete cluster --name ${KIND_CLUSTER_NAME}
 
+image-registry: ## Setup in-cluster image registry 
+	./tools/imageregistry/setup_imageregistry.sh ${KIND_CLUSTER_NAME}
+
 ###################
 # Install and Run #
 ###################
@@ -128,7 +131,7 @@ uninstall: ## Remove all rukpak resources from the cluster
 ##################
 # Build and Load #
 ##################
-.PHONY: build plain unpack core rukpakctl build-container kind-load kind-load-bundles kind-cluster
+.PHONY: build plain unpack core rukpakctl build-container kind-load kind-load-bundles kind-cluster registry-load-bundles
 
 ##@ build/load:
 
@@ -180,6 +183,9 @@ kind-load-bundles: kind ## Load the e2e testdata container images into a kind cl
 kind-load: kind ## Loads the currently constructed image onto the cluster
 	$(KIND) load docker-image $(IMAGE) --name $(KIND_CLUSTER_NAME)
 
+registry-load-bundles: kind-load-bundles ## Load the e2e testdata container images into registry
+	./tools/imageregistry/load_test_image.sh
+
 ###########
 # Release #
 ###########

tools/imageregistry/bundle_local_image.yaml

@@ -0,0 +1,11 @@
+apiVersion: core.rukpak.io/v1alpha1
+kind: Bundle
+metadata:
+  name: combo-v0.0.1
+spec:
+  source:
+    type: image
+    image:
+      ref: docker-registry.rukpak-e2e.svc.cluster.local:5000/bundles/plain-v0:valid
+      pullSecret: registrysecret
+  provisionerClassName: core.rukpak.io/plain

tools/imageregistry/daemonset.yaml

@@ -0,0 +1,46 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: setup-script
+data:
+  setup.sh: |
+    echo "$TRUSTED_CERT" > /usr/local/share/ca-certificates/ca.crt && update-ca-certificates && systemctl restart containerd
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: node-custom-setup
+  labels:
+    k8s-app: node-custom-setup
+spec:
+  selector:
+    matchLabels:
+      k8s-app: node-custom-setup
+  template:
+    metadata:
+      labels:
+        k8s-app: node-custom-setup
+    spec:
+      hostPID: true
+      hostNetwork: true
+      initContainers:
+      - name: init-node
+        command: ["nsenter"]
+        args: ["--mount=/proc/1/ns/mnt", "--", "sh", "-c", "$(SETUP_SCRIPT)"]
+        image: debian
+        env:
+        - name: TRUSTED_CERT
+          valueFrom:
+            configMapKeyRef:
+              name: trusted-ca
+              key: ca.crt
+        - name: SETUP_SCRIPT
+          valueFrom:
+            configMapKeyRef:
+              name: setup-script
+              key: setup.sh
+        securityContext:
+          privileged: true
+      containers:
+      - name: wait
+        image: k8s.gcr.io/pause:3.1

tools/imageregistry/load_test_image.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+export REGISTRY_NAME="docker-registry"
+export REGISTRY_NAMESPACE=rukpak-e2e
+export DNS_NAME=$REGISTRY_NAME.$REGISTRY_NAMESPACE.svc.cluster.local
+
+# push test bundle image into in-cluster docker registry
+kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl login -u myuser -p mypasswd $DNS_NAME:5000 --insecure-registry"
+
+for x in $(docker images --format "{{.Repository}}:{{.Tag}}" | grep testdata); do
+    kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io tag $x $DNS_NAME:5000${x##testdata}"
+    kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io push $DNS_NAME:5000${x##testdata} --insecure-registry"
+    kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io rmi $DNS_NAME:5000${x##testdata} --insecure-registry"
+done
+

tools/imageregistry/nerdctl.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nerdctl
+spec:
+  containers:
+  - command:
+    - sleep
+    - infinity
+    image: ghcr.io/containerd/nerdctl
+    imagePullPolicy: Always
+    name: nerdctl
+    volumeMounts:
+    - mountPath: /run/containerd
+      name: run-containerd
+    - mountPath: /var/lib/containerd
+      name: var-lib-containerd
+  volumes:
+  - name: run-containerd
+    hostPath:
+      path: /run/containerd
+  - name: var-lib-containerd
+    hostPath:
+      path: /var/lib/containerd

tools/imageregistry/registry.yaml

@@ -0,0 +1,49 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: docker-registry-pod
+  labels:
+    app: registry
+spec:
+  initContainers:
+  - name: auth
+    image: registry:2.6.2
+    command:
+    - "sh"
+    - "-c"
+    - "htpasswd -Bbn myuser mypasswd >> /auth/htpasswd"
+    volumeMounts:
+    - name: auth-vol
+      mountPath: "/auth"
+  containers:
+  - name: registry
+    image: registry:2.6.2
+    volumeMounts:
+    - name: repo-vol
+      mountPath: "/var/lib/registry"
+    - name: certs-vol
+      mountPath: "/certs"
+      readOnly: true
+    - name: auth-vol
+      mountPath: "/auth"
+      readOnly: true
+    env:
+    - name: REGISTRY_AUTH
+      value: "htpasswd"
+    - name: REGISTRY_AUTH_HTPASSWD_REALM
+      value: "Registry Realm"
+    - name: REGISTRY_AUTH_HTPASSWD_PATH
+      value: "/auth/htpasswd"
+    - name: REGISTRY_HTTP_TLS_CERTIFICATE
+      value: "/certs/tls.crt"
+    - name: REGISTRY_HTTP_TLS_KEY
+      value: "/certs/tls.key"
+  volumes:
+  - name: repo-vol
+    emptyDir: {}
+  - name: certs-vol
+    secret:
+      secretName: certs-secret
+  - name: auth-vol
+    emptyDir: {}
+

tools/imageregistry/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: docker-registry
+spec:
+  selector:
+    app: registry
+  ports:
+  - port: 5000
+    targetPort: 5000

tools/imageregistry/setup_imageregistry.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+export REGISTRY_NAME="docker-registry"
+export REGISTRY_NAMESPACE=rukpak-e2e
+export DNS_NAME=$REGISTRY_NAME.$REGISTRY_NAMESPACE.svc.cluster.local
+export KIND_CLUSTER_NAME=$1
+
+kubectl create ns $REGISTRY_NAMESPACE || true
+
+# create self-signed certificate for registry server
+mkdir -p /tmp/var/imageregistry/certs
+openssl req -x509 -newkey rsa:4096 -days 365 -nodes -sha256 -keyout /tmp/var/imageregistry/certs/tls.key -out /tmp/var/imageregistry/certs/tls.crt -subj "/CN=$DNS_NAME" -addext "subjectAltName = DNS:$DNS_NAME"
+kubectl create secret tls certs-secret --cert=/tmp/var/imageregistry/certs/tls.crt --key=/tmp/var/imageregistry/certs/tls.key -n $REGISTRY_NAMESPACE
+kubectl create configmap trusted-ca -n $REGISTRY_NAMESPACE --from-file=ca.crt=/tmp/var/imageregistry/certs/tls.crt
+
+# create image registry service
+kubectl apply -f tools/imageregistry/service.yaml -n $REGISTRY_NAMESPACE
+
+# set local variables
+export REGISTRY_IP=$(kubectl get service $REGISTRY_NAME -n $REGISTRY_NAMESPACE -o jsonpath='{ .spec.clusterIP }')
+export REGISTRY_PORT=5000
+
+# Add ca certificate to Node
+kubectl apply -f tools/imageregistry/daemonset.yaml -n $REGISTRY_NAMESPACE
+
+# Add an entry in /etc/hosts of Node
+docker exec $(docker ps | grep $KIND_CLUSTER_NAME'-control-plane' | cut -c 1-12) sh -c "/usr/bin/echo $REGISTRY_IP $DNS_NAME >>/etc/hosts"
+
+sleep 5
+# create image registry pod
+kubectl apply -f tools/imageregistry/registry.yaml -n $REGISTRY_NAMESPACE
+
+# create image upload  pod
+kubectl apply -f tools/imageregistry/nerdctl.yaml -n $REGISTRY_NAMESPACE
+
+# create imagePull secret for provisioner
+export IMAGE_PULL_RECRET="registrysecret"
+kubectl create ns rukpak-system || true
+kubectl create secret docker-registry $IMAGE_PULL_RECRET --docker-server=$DNS_NAME:5000 --docker-username="myuser" --docker-password="mypasswd" --docker-email="[email protected]" -n rukpak-system
+
+echo #### Valiables ####
+echo
+echo REGISTRY_NAME     $REGISTRY_NAME
+echo REGISTRY_IP       $REGISTRY_IP
+echo REGISTRY_PORT     $REGISTRY_PORT
+echo IMAGE_PULL_RECRET $IMAGE_PULL_RECRET
+
+# clean up 
+rm -rf /tmp/var/imageregistry/certs
+
+

tools/imageregistry/sniff.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+export REGISTRY_NAME="docker-registry"
+export REGISTRY_NAMESPACE=rukpak-e2e
+export DNS_NAME=$REGISTRY_NAME.$REGISTRY_NAMESPACE.svc.cluster.local
+export KIND_CLUSTER_NAME=$1
+
+# push test bundle image into in-cluster docker registry
+kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl login -u myuser -p mypasswd $DNS_NAME:5000 --insecure-registry"
+
+docker build testdata/bundles/plain-v0/valid -t testdata/bundles/plain-v0:valid
+kind load docker-image testdata/bundles/plain-v0:valid --name $KIND_CLUSTER_NAME
+kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io tag testdata/bundles/plain-v0:valid $DNS_NAME:5000/bundles/plain-v0:valid"
+kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io push $DNS_NAME:5000/bundles/plain-v0:valid --insecure-registry"
+kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io rmi $DNS_NAME:5000/bundles/plain-v0:valid --insecure-registry"
+
+# create bundle
+kubectl apply -f tools/imageregistry/bundle_local_image.yaml