Bpfd integration (#121)

This sets up the integration with [bpfd](bpfd.dev)

Additionally it adds a deployment scenario to the integration tests so
that we run with and without bpfd.

To test it out locally 
```
1. Build Cluster: make build.cluster
2. Install bpfd: kubectl apply -k ./config/bpfd-install
3. Install Blixt with bpfd programs: kubectl apply -k ./config/bpfd
```

---------

Signed-off-by: astoycos <[email protected]>

.github/workflows/test.yaml

@@ -77,6 +77,15 @@ jobs:
         BLIXT_DATAPLANE_IMAGE: "ghcr.io/kong/blixt-dataplane"
         BLIXT_UDP_SERVER_IMAGE: "ghcr.io/kong/blixt-udp-test-server"
         TAG: "integration-tests"
+
+    - name: run integration tests with bpfd
+      run: make test.integration
+      env:
+        BLIXT_CONTROLPLANE_IMAGE: "ghcr.io/kong/blixt-controlplane"
+        BLIXT_DATAPLANE_IMAGE: "ghcr.io/kong/blixt-dataplane"
+        BLIXT_UDP_SERVER_IMAGE: "ghcr.io/kong/blixt-udp-test-server"
+        BLIXT_USE_BPFD: true
+        TAG: "integration-tests"
 
     ## Upload diagnostics if integration test step failed.
     - name: upload diagnostics

Makefile

@@ -188,6 +188,10 @@ ifndef ignore-not-found
   ignore-not-found = false
 endif
 
+.PHONY: install-bpfd
+install-bpfd: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/bpfd | kubectl apply -f -
+
 .PHONY: install
 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/crd | kubectl apply -f -
@@ -316,3 +320,10 @@ build.cluster: $(KTF) # builds a KIND cluster which can be used for testing and
 load.image: build.image
 	kind load docker-image $(BLIXT_CONTROLPLANE_IMAGE):$(TAG) --name $(KIND_CLUSTER) && \
 		kubectl -n blixt-system rollout restart deployment blixt-controlplane
+
+.PHONY: load.all.images
+load.all.images: build.all.images
+	kind load docker-image $(BLIXT_CONTROLPLANE_IMAGE):$(TAG) --name $(KIND_CLUSTER) && \
+	kind load docker-image $(BLIXT_DATAPLANE_IMAGE):$(TAG) --name $(KIND_CLUSTER) && \
+	kind load docker-image $(BLIXT_UDP_SERVER_IMAGE):$(TAG) --name $(KIND_CLUSTER) && \
+		kubectl -n blixt-system rollout restart deployment blixt-controlplane

config/bpf-bytecode/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- tc-ingress.yaml
+- tc-egress.yaml

config/bpf-bytecode/tc-egress.yaml

@@ -0,0 +1,21 @@
+apiVersion: bpfd.dev/v1alpha1
+kind: TcProgram
+metadata:
+  labels:
+    app.kubernetes.io/name: blixt-tc-egress
+  name: tc-egress
+spec:
+  bpffunctionname: tc_egress
+  # Select all nodes
+  nodeselector: {}
+  interfaceselector:
+    primarynodeinterface: true
+  priority: 0
+  direction: egress
+  bytecode:
+    image:
+      url: quay.io/bpfd-bytecode/blixt-tc-egress:latest
+      imagepullpolicy: Always
+  mapownerselector:
+    matchLabels:
+      bpfd.dev/ownedByProgram: blixt-tc-ingress

config/bpf-bytecode/tc-ingress.yaml

@@ -0,0 +1,18 @@
+apiVersion: bpfd.dev/v1alpha1
+kind: TcProgram
+metadata:
+  labels:
+    app.kubernetes.io/name: blixt-tc-ingress
+  name: tc-ingress
+spec:
+  bpffunctionname: tc_ingress
+  # Select all nodes
+  nodeselector: {}
+  interfaceselector:
+    primarynodeinterface: true
+  priority: 0
+  direction: ingress
+  bytecode:
+    image:
+      url: quay.io/bpfd-bytecode/blixt-tc-ingress:latest
+      imagepullpolicy: Always

config/bpfd-install/kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+## bpfd CRDs
+- https://github.com/bpfd-dev/bpfd/releases/download/v0.3.0/bpfd-crds-install-v0.3.0.yaml
+## bpfd Operator
+- https://github.com/bpfd-dev/bpfd/releases/download/v0.3.0/bpfd-operator-install-v0.3.0.yaml
+
+patches:
+- path: patch.yaml
+  target:
+    kind: ConfigMap
+    name: config
+    version: v1
+    name: bpfd-config

config/bpfd-install/patch.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config
+data:
+  ## Can be configured at runtime
+  bpfd.log.level: "debug"
+  bpfd.agent.log.level: "debug"
+  bpfd.enable.csi: "true"
+  ## Custom temporary blixt build
+  bpfd.image: quay.io/bpfd/bpfd:blixt

config/bpfd/bpfd.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: dataplane
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: dataplane
+        securityContext:
+          privileged: false
+        volumeMounts:
+        - name: bpf-maps
+          mountPath: /run/bpfd/fs/maps
+          readOnly: true
+      volumes:
+      - name: bpf-maps
+        csi:
+          driver: csi.bpfd.dev
+          volumeAttributes:
+            csi.bpfd.dev/program: blixt-tc-egress
+            csi.bpfd.dev/maps: AYA_LOGS,AYA_LOG_BUF,BACKENDS,BLIXT_CONNTRACK,GATEWAY_INDEXES

config/bpfd/kustomization.yaml

@@ -0,0 +1,50 @@
+# Adds namespace to all resources.
+namespace: blixt-system
+
+# Value of this field is prepended to the
+# names of all resources, e.g. a deployment named
+# "wordpress" becomes "alices-wordpress".
+# Note that it should also match with the prefix (text before '-') of the namespace
+# field above.
+namePrefix: blixt-
+
+# Labels to add to all resources and selectors.
+#commonLabels:
+#  someName: someValue
+
+# - ../crd # TODO: no CRDs yet
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
+#- ../webhook
+# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
+#- ../certmanager
+# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
+#- ../prometheus
+
+# Protect the /metrics endpoint by putting it behind auth.
+# If you want your controlplane to expose the /metrics
+# endpoint w/o any authn/z, please comment the following line.
+
+# Mount the controller config file for loading manager configurations
+# through a ComponentConfig type
+#- manager_config_patch.yaml
+
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
+#- manager_webhook_patch.yaml
+
+# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
+# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
+# 'CERTMANAGER' needs to be enabled to use ca injection
+#- webhookcainjection_patch.yaml
+
+# the following config is for teaching kustomize how to do var substitution
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../bpf-bytecode
+- ../rbac
+- ../manager
+- ../dataplane
+patches:
+- path: bpfd.yaml

config/tests/integration-bpfd/kustomization.yaml

@@ -0,0 +1,12 @@
+
+images:
+- name: ghcr.io/kong/blixt-dataplane
+  newTag: integration-tests
+- name: ghcr.io/kong/blixt-controlplane
+  newTag: integration-tests
+- name: ghcr.io/kong/blixt-udp-test-server
+  newTag: integration-tests
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../bpfd

dataplane/.cargo/config.toml

@@ -1,2 +1,6 @@
 [alias]
-xtask = "run --package xtask --"
+xtask = "run --package xtask --"
+
+## Needed to build blixt's programs with BTF + custom bpf-linker build.
+[build]
+rustflags = ["-C", "debuginfo=2"]

dataplane/Makefile

@@ -26,3 +26,25 @@ build.image:
 load.image: build.image
 	kind load docker-image $(IMAGE):$(TAG) --name $(KIND_CLUSTER) && \
 		kubectl -n blixt-system rollout restart daemonset blixt-dataplane
+
+.PHONY: build.bytecode.images
+build.bytecode.images: build
+	docker build \
+      --build-arg PROGRAM_NAME=blixt-tc-ingress \
+      --build-arg BPF_FUNCTION_NAME=tc_ingress \
+      --build-arg PROGRAM_TYPE=tc \
+      --build-arg BYTECODE_FILENAME=loader \
+      -f https://raw.githubusercontent.com/bpfd-dev/bpfd/main/packaging/container-deployment/Containerfile.bytecode \
+	  ./target/bpfel-unknown-none/debug -t quay.io/bpfd-bytecode/blixt-tc-ingress:latest
+	docker build \
+      --build-arg PROGRAM_NAME=blixt-tc-egress \
+      --build-arg BPF_FUNCTION_NAME=tc_egress \
+      --build-arg PROGRAM_TYPE=tc \
+      --build-arg BYTECODE_FILENAME=loader \
+      -f https://raw.githubusercontent.com/bpfd-dev/bpfd/main/packaging/container-deployment/Containerfile.bytecode \
+	  ./target/bpfel-unknown-none/debug -t quay.io/bpfd-bytecode/blixt-tc-egress:latest
+
+.PHONY: push.bytecode.images
+push.bytecode.images: build.bytecode.images
+	docker push quay.io/bpfd-bytecode/blixt-tc-egress:latest
+	docker push quay.io/bpfd-bytecode/blixt-tc-ingress:latest