Include peerpod specific crd deployments via operator

This includes peerpod and peerpodconfig controllers

Signed-off-by: Cathy Avery <[email protected]>
Signed-off-by: Pradipta Banerjee <[email protected]>

Dockerfile.peerpods

@@ -0,0 +1,39 @@
+# Golang image and version (defaults are provided).
+# Use e.g. `golang` for multi-arch support.
+ARG IMG_NAME
+ARG IMG_VERSION
+
+# Build the manager binary
+FROM ${IMG_NAME:-golang}:${IMG_VERSION:-1.20} as builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the go source
+COPY main.go main.go
+COPY api/ api/
+COPY controllers/ controllers/
+
+# The container running the operator/controllers needs the libvirt libraries as
+# as the peerpod-ctrl needs to dynamically link with libvirt
+RUN apt-get update -y && apt-get install -y libvirt-dev
+
+# Build
+RUN CGO_ENABLED=1 GOOS=linux go build -a -o manager main.go
+
+# Libvirt cannot be installed to distroless for packaging so remove it for now
+# Use distroless as minimal base image to package the manager binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+# FROM gcr.io/distroless/static:nonroot
+
+FROM ${IMG_NAME:-golang}:${IMG_VERSION:-1.20}
+RUN apt-get update -y && apt-get install -y libvirt-dev
+WORKDIR /
+COPY --from=builder /workspace/manager .
+
+ENTRYPOINT ["/manager"]

Makefile

@@ -141,7 +141,12 @@ run: manifests generate fmt vet ## Run a controller from your host.
 
 .PHONY: docker-build
 docker-build: test ## Build docker image with the manager.
+ifneq (, $(PEERPODS))
+	@echo PEERPODS is enabled
+	docker build -t ${IMG} -f Dockerfile.peerpods .
+else
 	docker build -t ${IMG} .
+endif
 
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.
@@ -156,19 +161,33 @@ endif
 .PHONY: install
 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/crd | kubectl apply -f -
+ifneq (, $(PEERPODS))
+	$(KUSTOMIZE) build config/overlays/peerpods/crd | kubectl apply -f -
+endif
 
 .PHONY: uninstall
 uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
+ifneq (, $(PEERPODS))
+	$(KUSTOMIZE) build config/overlays/peerpods/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
+endif
 
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+ifneq (, $(PEERPODS))
+	$(KUSTOMIZE) build config/overlays/peerpods/default | kubectl apply -f -
+else
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
+endif
 
 .PHONY: undeploy
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+ifneq (, $(PEERPODS))
+	$(KUSTOMIZE) build config/overlays/peerpods/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
+else
 	$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
+endif
 
 ##@ Build Dependencies
 

PROJECT

@@ -14,4 +14,12 @@ resources:
   kind: CcRuntime
   path: github.com/confidential-containers/operator/api/v1beta1
   version: v1beta1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: confidentialcontainers.org
+  kind: PeerPod
+  path: github.com/confidential-containers/cloud-api-adaptor/peerpod-ctrl/api/v1alpha1
+  version: v1alpha1
 version: "3"

config/overlays/peerpods/crd/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- github.com/confidential-containers/cloud-api-adaptor//peerpodconfig-ctrl/config/crd?ref=v0.8.0
+- github.com/confidential-containers/cloud-api-adaptor//peerpod-ctrl/config/crd?ref=v0.8.0
+#+kubebuilder:scaffold:crdkustomizeresource

config/overlays/peerpods/default/caa_rbac/caa_rbac.yaml

@@ -0,0 +1,29 @@
+# This file is based on https://github.com/confidential-containers/cloud-api-adaptor/blob/staging/install/rbac/peer-pod.yaml
+# It adds the required rules to the default SA which is used by CAA DA
+# when owner reference is both object must be on the same namespace, hence,
+# caa should have cluster-wide permissions to support any pod namespace
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: caa-role
+rules:
+- apiGroups: [""]
+  resources: ["pods", "pods/finalizers"]
+  verbs: ["get","create", "patch", "update"]
+- apiGroups: ["confidentialcontainers.org"]
+  resources: ["peerpods", "pods"]
+  verbs: ["create", "patch", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: caa-rolebinding
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: confidential-containers-system
+roleRef:
+  kind: ClusterRole
+  name: caa-role
+  apiGroup: rbac.authorization.k8s.io

config/overlays/peerpods/default/caa_rbac/kustomization.yaml

@@ -0,0 +1,4 @@
+namePrefix: cc-operator-
+
+resources:
+- caa_rbac.yaml

config/overlays/peerpods/default/enable-peerpods.yaml

@@ -0,0 +1,3 @@
+ - op: add
+   path: /spec/template/spec/containers/0/args/-
+   value: --peer-pods

config/overlays/peerpods/default/kustomization.yaml

@@ -0,0 +1,28 @@
+resources:
+- ../../../default
+- ../crd
+- ./caa_rbac
+
+patches:
+- path: runasnonroot.yaml
+- path: ssh-volume.yaml
+- path: ssh-mount.yaml
+- path: peerpods-ctrl-role.yaml
+  target:
+    group: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: manager-role
+    version: v1
+- path: peerpodsconfig-ctrl-role.yaml
+  target:
+    group: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: manager-role
+    version: v1
+- path: peerpods-namespace.yaml
+- path: enable-peerpods.yaml
+  target:
+    group: apps
+    kind: Deployment
+    name: controller-manager
+    version: v1

config/overlays/peerpods/default/peerpods-ctrl-role.yaml

@@ -0,0 +1,35 @@
+- op: add
+  path: /rules/0
+  value:
+    apiGroups:
+      - confidentialcontainers.org
+    resources:
+      - peerpods
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+- op: add
+  path: /rules/0
+  value:
+    apiGroups:
+      - confidentialcontainers.org
+    resources:
+      - peerpods/finalizers
+    verbs:
+      - update
+- op: add
+  path: /rules/0
+  value:
+    apiGroups:
+      - confidentialcontainers.org
+    resources:
+      - peerpods/status
+    verbs:
+      - get
+      - patch
+      - update

config/overlays/peerpods/default/peerpods-namespace.yaml

@@ -0,0 +1,13 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        env:
+        - name: PEERPODS_NAMESPACE
+          value: "confidential-containers-system"

config/overlays/peerpods/default/peerpodsconfig-ctrl-role.yaml

@@ -0,0 +1,70 @@
+- op: add
+  path: /rules/0
+  value:
+    apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - create
+      - get
+      - list
+      - update
+      - watch
+- op: add
+  path: /rules/0
+  value:
+    apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch
+- op: add
+  path: /rules/0
+  value:
+    apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - create
+      - get
+      - list
+      - update
+      - watch
+- op: add
+  path: /rules/0
+  value:
+    apiGroups:
+      - confidentialcontainers.org
+    resources:
+      - peerpodconfigs
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+- op: add
+  path: /rules/0
+  value:
+    apiGroups:
+      - confidentialcontainers.org
+    resources:
+      - peerpodconfigs/finalizers
+    verbs:
+      - update
+- op: add
+  path: /rules/0
+  value:
+    apiGroups:
+      - confidentialcontainers.org
+    resources:
+      - peerpodconfigs/status
+    verbs:
+      - get
+      - patch
+      - update

config/overlays/peerpods/default/runasnonroot.yaml

@@ -0,0 +1,10 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      securityContext:
+        runAsNonRoot: false

config/overlays/peerpods/default/ssh-mount.yaml

@@ -0,0 +1,14 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        volumeMounts:
+        - mountPath: /root/.ssh/
+          name: ssh
+          readOnly: true

config/overlays/peerpods/default/ssh-volume.yaml

@@ -0,0 +1,14 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      volumes:
+      - name: ssh
+        secret:
+          defaultMode: 384
+          optional: true
+          secretName: ssh-key-secret

config/samples/ccruntime/peer-pods/kustomization.yaml

@@ -5,6 +5,7 @@ nameSuffix: -peer-pods
 
 resources:
 - ../base
+- peerpodsconfig.yaml
 
 images:
 - name: quay.io/confidential-containers/reqs-payload

config/samples/ccruntime/peer-pods/peerpodsconfig.yaml

@@ -0,0 +1,9 @@
+apiVersion: confidentialcontainers.org/v1alpha1
+kind: PeerPodConfig
+metadata:
+  name: coco-config
+  namespace: confidential-containers-system
+spec:
+  cloudSecretName: "peer-pods-secret"
+  configMapName:   "peer-pods-cm"
+  limit: "1"