From 92eb08ae767e9c6431fbcb8a16cfa9dd0667e227 Mon Sep 17 00:00:00 2001 From: Cathy Avery Date: Fri, 4 Aug 2023 10:01:10 -0400 Subject: [PATCH] add peerpod controller Fixes: #872 Signed-off-by: Cathy Avery --- Dockerfile.peerpods-libvirt | 39 ++++++++ Makefile | 5 + PROJECT | 8 ++ config/crd/kustomization.yaml | 1 + config/default/kustomization.yaml | 3 + config/default/peerpods/runasnonroot.yaml | 10 ++ config/default/peerpods/ssh-mount.yaml | 14 +++ config/default/peerpods/ssh-volume.yaml | 14 +++ config/rbac/caa_rbac.yaml | 29 ++++++ config/rbac/ccruntime_editor_role.yaml | 2 + config/rbac/ccruntime_viewer_role.yaml | 2 + config/rbac/kustomization.yaml | 3 + config/rbac/role.yaml | 26 +++++ controllers/ccruntime_controller.go | 3 + docs/PEERPODS-TEST.md | 112 ++++++++++++++++++++++ docs/PEERPODS.md | 82 ++++++++++++++++ main.go | 15 +++ 17 files changed, 368 insertions(+) create mode 100644 Dockerfile.peerpods-libvirt create mode 100644 config/default/peerpods/runasnonroot.yaml create mode 100644 config/default/peerpods/ssh-mount.yaml create mode 100644 config/default/peerpods/ssh-volume.yaml create mode 100644 config/rbac/caa_rbac.yaml create mode 100644 docs/PEERPODS-TEST.md create mode 100644 docs/PEERPODS.md diff --git a/Dockerfile.peerpods-libvirt b/Dockerfile.peerpods-libvirt new file mode 100644 index 00000000..0b3f6569 --- /dev/null +++ b/Dockerfile.peerpods-libvirt @@ -0,0 +1,39 @@ +# Golang image and version (defaults are provided). +# Use e.g. `golang` for multi-arch support. +ARG IMG_NAME +ARG IMG_VERSION + +# Build the manager binary +FROM ${IMG_NAME:-golang}:${IMG_VERSION:-1.20} as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY main.go main.go +COPY api/ api/ +COPY controllers/ controllers/ + +# The container running the operator/controllers needs the libvirt libraries as +# as the peerpod-ctrl needs to dynamically link with libvirt +RUN apt-get update -y && apt-get install -y libvirt-dev + +# Build +RUN CGO_ENABLED=1 GOOS=linux go build -tags=libvirt -a -o manager main.go + +# Libvirt cannot be installed to distroless for packaging so remove it for now +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +# FROM gcr.io/distroless/static:nonroot + +FROM ${IMG_NAME:-golang}:${IMG_VERSION:-1.20} +RUN apt-get update -y && apt-get install -y libvirt-dev +WORKDIR / +COPY --from=builder /workspace/manager . + +ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index 4e1ed449..67db9c56 100644 --- a/Makefile +++ b/Makefile @@ -141,7 +141,12 @@ run: manifests generate fmt vet ## Run a controller from your host. .PHONY: docker-build docker-build: test ## Build docker image with the manager. +ifneq (, $(PEERPODS)) + @echo PEERPODS is enabled + docker build -t ${IMG} -f Dockerfile.peerpods-libvirt . +else docker build -t ${IMG} . +endif .PHONY: docker-push docker-push: ## Push docker image with the manager. diff --git a/PROJECT b/PROJECT index 544a6e35..68eec626 100644 --- a/PROJECT +++ b/PROJECT @@ -14,4 +14,12 @@ resources: kind: CcRuntime path: github.com/confidential-containers/operator/api/v1beta1 version: v1beta1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: confidentialcontainers.org + kind: PeerPod + path: github.com/confidential-containers/cloud-api-adaptor/peerpod-ctrl/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 00e48ec3..585ee2e4 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -7,6 +7,7 @@ kind: Kustomization resources: - bases/confidentialcontainers.org_ccruntimes.yaml - github.com/confidential-containers/cloud-api-adaptor//peerpodconfig-ctrl/config/crd?ref=v0.7.0 +- github.com/confidential-containers/cloud-api-adaptor//peerpod-ctrl/config/crd?ref=v0.7.0 #+kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index e7450d2a..0c74dacb 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -38,6 +38,9 @@ patches: #- manager_config_patch.yaml # [PEERPODS] To enable the peerpod controllers uncomment this section. +#- path: peerpods/runasnonroot.yaml +#- path: peerpods/ssh-volume.yaml +#- path: peerpods/ssh-mount.yaml #- path: peerpods/peerpods-namespace.yaml #- path: peerpods/enable-peerpods.yaml # target: diff --git a/config/default/peerpods/runasnonroot.yaml b/config/default/peerpods/runasnonroot.yaml new file mode 100644 index 00000000..cfaa9dc3 --- /dev/null +++ b/config/default/peerpods/runasnonroot.yaml @@ -0,0 +1,10 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + securityContext: + runAsNonRoot: false diff --git a/config/default/peerpods/ssh-mount.yaml b/config/default/peerpods/ssh-mount.yaml new file mode 100644 index 00000000..767cfdfe --- /dev/null +++ b/config/default/peerpods/ssh-mount.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /root/.ssh/ + name: ssh + readOnly: true diff --git a/config/default/peerpods/ssh-volume.yaml b/config/default/peerpods/ssh-volume.yaml new file mode 100644 index 00000000..551d5624 --- /dev/null +++ b/config/default/peerpods/ssh-volume.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + volumes: + - name: ssh + secret: + defaultMode: 384 + optional: true + secretName: ssh-key-secret diff --git a/config/rbac/caa_rbac.yaml b/config/rbac/caa_rbac.yaml new file mode 100644 index 00000000..0525a910 --- /dev/null +++ b/config/rbac/caa_rbac.yaml @@ -0,0 +1,29 @@ +# This file is based on https://github.com/confidential-containers/cloud-api-adaptor/blob/staging/install/rbac/peer-pod.yaml +# It adds the required rules to the default SA which is used by CAA DA +# when owner reference is both object must be on the same namespace, hence, +# caa should have cluster-wide permissions to support any pod namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: caa-role +rules: +- apiGroups: [""] + resources: ["pods", "pods/finalizers"] + verbs: ["get","create", "patch", "update"] +- apiGroups: ["confidentialcontainers.org"] + resources: ["peerpods", "pods"] + verbs: ["create", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: caa-rolebinding +subjects: +- kind: ServiceAccount + name: default + namespace: system +roleRef: + kind: ClusterRole + name: caa-role + apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/ccruntime_editor_role.yaml b/config/rbac/ccruntime_editor_role.yaml index 9ad4690e..e0db2865 100644 --- a/config/rbac/ccruntime_editor_role.yaml +++ b/config/rbac/ccruntime_editor_role.yaml @@ -10,6 +10,7 @@ rules: resources: - ccruntimes - peerpodconfigs + - peerpods verbs: - create - delete @@ -24,5 +25,6 @@ rules: resources: - ccruntimes/status - peerpodconfigs/status + - peerpods/status verbs: - get diff --git a/config/rbac/ccruntime_viewer_role.yaml b/config/rbac/ccruntime_viewer_role.yaml index 3f0c558c..94b4a92c 100644 --- a/config/rbac/ccruntime_viewer_role.yaml +++ b/config/rbac/ccruntime_viewer_role.yaml @@ -10,6 +10,7 @@ rules: resources: - ccruntimes - peerpodconfigs + - peerpods verbs: - get - list @@ -20,5 +21,6 @@ rules: resources: - ccruntimes/status - peerpodconfigs/status + - peerpods/status verbs: - get diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 731832a6..6407644e 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -16,3 +16,6 @@ resources: - auth_proxy_role.yaml - auth_proxy_role_binding.yaml - auth_proxy_client_clusterrole.yaml +# the following is custom rbac manifests required for +# cloud-api-adaptor when peerpod-ctrl is used +- caa_rbac.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 7d960bd9..b934ac9c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -102,6 +102,32 @@ rules: - get - patch - update +- apiGroups: + - confidentialcontainers.org + resources: + - peerpods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - confidentialcontainers.org + resources: + - peerpods/finalizers + verbs: + - update +- apiGroups: + - confidentialcontainers.org + resources: + - peerpods/status + verbs: + - get + - patch + - update - apiGroups: - "" resources: diff --git a/controllers/ccruntime_controller.go b/controllers/ccruntime_controller.go index 8dd46635..3ca350db 100644 --- a/controllers/ccruntime_controller.go +++ b/controllers/ccruntime_controller.go @@ -73,6 +73,9 @@ const ( //+kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpodconfigs,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpodconfigs/status,verbs=get;update;patch //+kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpodconfigs/finalizers,verbs=update +//+kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpods,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpods/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=confidentialcontainers.org,resources=peerpods/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=configmaps,verbs=create;get;update;list;watch //+kubebuilder:rbac:groups="",resources=secrets,verbs=create;get;update;list;watch diff --git a/docs/PEERPODS-TEST.md b/docs/PEERPODS-TEST.md new file mode 100644 index 00000000..156c0339 --- /dev/null +++ b/docs/PEERPODS-TEST.md @@ -0,0 +1,112 @@ +## Introduction +This document describes testing to confirm that the operators are functioning. +These test were performed on a libvirt provider as an example. + +## Check that the peerpod-config CR was created +``` +kubectl get peerpodconfigs -n confidential-containers-system +NAME AGE +coco-config-peer-pods 15m +``` + +## Check that the peerpodconfig-ctrl controller created the cloud adapter daemon and it is running +``` +kubectl get pods -n confidential-containers-system +NAME READY STATUS RESTARTS AGE +cc-operator-controller-manager-68b5979488-x8bhb 2/2 Running 0 31m +cc-operator-daemon-install-lrwc5 1/1 Running 0 12s +cc-operator-pre-install-daemon-2cllk 1/1 Running 0 14s +peerpodconfig-ctrl-caa-daemon-bv45g 1/1 Running 0 14s +``` + +## Create a Peerpod +``` +kubectl apply -f fedora-sleep.yaml +``` +``` +fedora-sleep.yaml +apiVersion: v1 +kind: Pod +metadata: + name: fedora-sleep +spec: + runtimeClassName: kata-remote + restartPolicy: Never + containers: + - name: sleep-forever + image: registry.fedoraproject.org/fedora + command: ["sleep"] + args: [ "infinity"] + +``` + +## Check that the peerpod CR was created as a result of creating the peerpod +``` +kubectl get peerpods +NAME AGE +fedora-sleep-resource-gqjl6 7m26s +``` + +## Check that the peerpod pod is running and that a VM was created and is up +``` +kubectl get pods +NAME READY STATUS RESTARTS AGE +fedora-sleep 1/1 Running 0 7m7s + +kcli get vms ++-----------------------------+--------+-----------------+------------+----------+---------+ +| Name | Status | Ip | Source | Plan | Profile | ++-----------------------------+--------+-----------------+------------+----------+---------+ +| coco-k8s-ctlplane-0 | up | 192.168.122.251 | ubuntu2004 | coco-k8s | kvirt | +| coco-k8s-worker-0 | up | 192.168.122.68 | ubuntu2004 | coco-k8s | kvirt | +| podvm-fedora-sleep-59d239f2 | up | 192.168.122.23 | | | | ++-----------------------------+--------+-----------------+------------+----------+---------+ +``` + +## Check that the peerpod-ctrl controller is working +First delete the cloud adapter daemon. +``` +kubectl delete pod peerpodconfig-ctrl-caa-daemon-bv45g -n confidential-containers-system +pod "peerpodconfig-ctrl-caa-daemon-bv45g" deleted +``` + +This will cause the peerpod to error. +``` +kubectl get pods +NAME READY STATUS RESTARTS AGE +fedora-sleep 0/1 Error 0 13m +``` +A new caa-daemon will start up but it will not know about the existing peerpod +VM. The peerpod-ctrl controller will clean up the orphaned peerpod resources once the pod +is deleted. +``` +kubectl delete pod fedora-sleep +pod "fedora-sleep" deleted + +kubectl get peerpod +No resources found in default namespace. + +kcli get vms ++---------------------+--------+-----------------+------------+----------+---------+ +| Name | Status | Ip | Source | Plan | Profile | ++---------------------+--------+-----------------+------------+----------+---------+ +| coco-k8s-ctlplane-0 | up | 192.168.122.251 | ubuntu2004 | coco-k8s | kvirt | +| coco-k8s-worker-0 | up | 192.168.122.68 | ubuntu2004 | coco-k8s | kvirt | ++---------------------+--------+-----------------+------------+----------+---------+ +``` + +In the operator log look for [adaptor/cloud/libvirt] and you should see references to deleting the instance resources: +``` +kubectl logs cc-operator-controller-manager-68b5979488-x8bhb -f -n confidential-containers-system --all-containers=true + +INFO deleting instance {"controller": "peerpod", "controllerGroup": "confidentialcontainers.org", "controllerKind": "PeerPod", "PeerPod": {"name":"fedora-sleep-resource-gqjl6","namespace":"default"}, "namespace": "default", "name": "fedora-sleep-resource-gqjl6", "reconcileID": "520379c0-e37a-4628-9c85-67955502a7d0", "InstanceID": "42", "CloudProvider": "libvirt"} +[adaptor/cloud/libvirt] Deleting instance (42) +[adaptor/cloud/libvirt] Checking if instance (42) exists +[adaptor/cloud/libvirt] domainDef [{{ disk} disk 0xc00096f8c0 0xc0006ac870 0xc0000d7f90 0xc0005c41f8 0xc0005b5200} {{ disk} cdrom 0xc00096fe60 0xc0006ac900 0xc00088c000 0x308fd88 0xc0005b5440}] +[adaptor/cloud/libvirt] Check if podvm-fedora-sleep-59d239f2-root.qcow2 volume exists +[adaptor/cloud/libvirt] Deleting volume podvm-fedora-sleep-59d239f2-root.qcow2 +[adaptor/cloud/libvirt] Check if podvm-fedora-sleep-59d239f2-cloudinit.iso volume exists +[adaptor/cloud/libvirt] Deleting volume podvm-fedora-sleep-59d239f2-cloudinit.iso +[adaptor/cloud/libvirt] deleted an instance 42 +INFO instance deleted {"controller": "peerpod", "controllerGroup": "confidentialcontainers.org", "controllerKind": "PeerPod", "PeerPod": {"name":"fedora-sleep-resource-gqjl6","namespace":"default"}, "namespace": "default", "name": "fedora-sleep-resource-gqjl6", "reconcileID": "520379c0-e37a-4628-9c85-67955502a7d0", "InstanceID": "42", "CloudProvider": "libvirt"} +``` diff --git a/docs/PEERPODS.md b/docs/PEERPODS.md new file mode 100644 index 00000000..92f3c85b --- /dev/null +++ b/docs/PEERPODS.md @@ -0,0 +1,82 @@ +# Introduction +These instructions outline how to build and install the operator with the peerpods +controllers. Please refer to the DEVELOPMENT.md and INSTALL.md guides for additional +instructions and prerequisites. + +Please visit (https://github.com/confidential-containers/cloud-api-adaptor) for more +information on peerpods, peerpodconfig-ctrl, and peerpod-ctrl. + +## Enable Peerpods +Follow the instructions in config/default/kustomization.yaml regarding [PEERPODS]. +The kustomiztion.yaml peerpod patch overlays currently only support the libvirt provider. + +## Set Environment Variables +``` +export QUAY_USER= +export IMG=quay.io/${QUAY_USER}/cc-operator +export PEERPODS=1 +``` + +## Building Operator image +``` +make docker-build +make docker-push +``` + +## Deploying Operator +Ensure KUBECONFIG points to target Kubernetes cluster +``` +make install && make deploy +``` + +## Apply the ssh secret needed for the libvirt connection. +``` +kubectl create secret generic ssh-key-secret -n confidential-containers-system --from-file=id_rsa.pub=./id_rsa.pub --from-file=id_rsa=./id_rsa +``` + +## Apply the peerpod secret and config map +``` +kubectl apply -f peer-pods-config-map.yaml +``` +peer-pods-config-map.yaml example +``` +apiVersion: v1 +kind: ConfigMap +metadata: + name: peer-pods-cm + namespace: confidential-containers-system +data: + CLOUD_PROVIDER: "libvirt" + PROXY_TIMEOUT: 30m +``` + +``` +kubectl apply -f peer-pods-secret.yaml +``` +peer-pods-secret.yaml example +``` +apiVersion: v1 +kind: Secret +metadata: + name: peer-pods-secret + namespace: confidential-containers-system +type: Opaque +stringData: + CLOUD_PROVIDER: "libvirt" + VXLAN_PORT: "9000" + LIBVIRT_URI: "qemu+ssh://root@192.168.122.1/system?no_verify=1" + LIBVIRT_NET: "default" + LIBVIRT_POOL: "default" +``` + +## Create Custom Resource (CR) +``` +kubectl create -k config/samples/ccruntime/peerpods +``` + +## Uninstalling Operator + +Ensure KUBECONFIG points to target Kubernetes cluster +``` +make uninstall && make undeploy +``` \ No newline at end of file diff --git a/main.go b/main.go index 2cf121bd..32613eb9 100644 --- a/main.go +++ b/main.go @@ -21,6 +21,7 @@ import ( "flag" "os" + peerpodcontrollers "github.com/confidential-containers/cloud-api-adaptor/peerpod-ctrl/controllers" peerpodconfigcontrollers "github.com/confidential-containers/cloud-api-adaptor/peerpodconfig-ctrl/controllers" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -38,6 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" + peerpod "github.com/confidential-containers/cloud-api-adaptor/peerpod-ctrl/api/v1alpha1" peerpodconfig "github.com/confidential-containers/cloud-api-adaptor/peerpodconfig-ctrl/api/v1alpha1" ccv1beta1 "github.com/confidential-containers/operator/api/v1beta1" @@ -56,6 +58,8 @@ func init() { utilruntime.Must(ccv1beta1.AddToScheme(scheme)) utilruntime.Must(peerpodconfig.AddToScheme(scheme)) + + utilruntime.Must(peerpod.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -123,6 +127,17 @@ func main() { setupLog.Error(err, "unable to create RemotePodConfig controller for OpenShift cluster", "controller", "RemotePodConfig") os.Exit(1) } + + if err = (&peerpodcontrollers.PeerPodReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + // setting nil will delegate Provider creation to reconcile time, make sure RBAC permits: + //+kubebuilder:rbac:groups="",resourceNames=peer-pods-cm;peer-pods-secret,resources=configmaps;secrets,verbs=get + Provider: nil, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create peerpod resources controller", "controller", "PeerPod") + os.Exit(1) + } } //+kubebuilder:scaffold:builder