bpfd-integration: add new deployment configs

Add new config to allow:
- deploying bpfd (with custom blixt settings)
- Run integrations on the blixt + bpfd setup
- Activate all of this in CI
- Allow building of new blixt program bytecode images

Signed-off-by: astoycos <[email protected]>

.github/workflows/test.yaml

@@ -77,6 +77,15 @@ jobs:
         BLIXT_DATAPLANE_IMAGE: "ghcr.io/kong/blixt-dataplane"
         BLIXT_UDP_SERVER_IMAGE: "ghcr.io/kong/blixt-udp-test-server"
         TAG: "integration-tests"
+
+    - name: run integration tests with bpfd
+      run: make test.integration
+      env:
+        BLIXT_CONTROLPLANE_IMAGE: "ghcr.io/kong/blixt-controlplane"
+        BLIXT_DATAPLANE_IMAGE: "ghcr.io/kong/blixt-dataplane"
+        BLIXT_UDP_SERVER_IMAGE: "ghcr.io/kong/blixt-udp-test-server"
+        BLIXT_USE_BPFD: true
+        TAG: "integration-tests"
 
     ## Upload diagnostics if integration test step failed.
     - name: upload diagnostics

Makefile

@@ -188,6 +188,10 @@ ifndef ignore-not-found
   ignore-not-found = false
 endif
 
+.PHONY: install-bpfd
+install-bpfd: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/bpfd | kubectl apply -f -
+
 .PHONY: install
 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/crd | kubectl apply -f -
@@ -316,3 +320,10 @@ build.cluster: $(KTF) # builds a KIND cluster which can be used for testing and
 load.image: build.image
 	kind load docker-image $(BLIXT_CONTROLPLANE_IMAGE):$(TAG) --name $(KIND_CLUSTER) && \
 		kubectl -n blixt-system rollout restart deployment blixt-controlplane
+
+.PHONY: load.all.images
+load.all.images: build.all.images
+	kind load docker-image $(BLIXT_CONTROLPLANE_IMAGE):$(TAG) --name $(KIND_CLUSTER) && \
+	kind load docker-image $(BLIXT_DATAPLANE_IMAGE):$(TAG) --name $(KIND_CLUSTER) && \
+	kind load docker-image $(BLIXT_UDP_SERVER_IMAGE):$(TAG) --name $(KIND_CLUSTER) && \
+		kubectl -n blixt-system rollout restart deployment blixt-controlplane

config/bpf-bytecode/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- tc-ingress.yaml
+- tc-egress.yaml

config/bpf-bytecode/tc-egress.yaml

@@ -0,0 +1,21 @@
+apiVersion: bpfd.dev/v1alpha1
+kind: TcProgram
+metadata:
+  labels:
+    app.kubernetes.io/name: blixt-tc-egress
+  name: tc-egress
+spec:
+  bpffunctionname: tc_egress
+  # Select all nodes
+  nodeselector: {}
+  interfaceselector:
+    primarynodeinterface: true
+  priority: 0
+  direction: egress
+  bytecode:
+    image:
+      url: quay.io/bpfd-bytecode/blixt-tc-egress:latest
+      imagepullpolicy: Always
+  mapownerselector:
+    matchLabels:
+      bpfd.dev/ownedByProgram: blixt-tc-ingress

config/bpf-bytecode/tc-ingress.yaml

@@ -0,0 +1,18 @@
+apiVersion: bpfd.dev/v1alpha1
+kind: TcProgram
+metadata:
+  labels:
+    app.kubernetes.io/name: blixt-tc-ingress
+  name: tc-ingress
+spec:
+  bpffunctionname: tc_ingress
+  # Select all nodes
+  nodeselector: {}
+  interfaceselector:
+    primarynodeinterface: true
+  priority: 0
+  direction: ingress
+  bytecode:
+    image:
+      url: quay.io/bpfd-bytecode/blixt-tc-ingress:latest
+      imagepullpolicy: Always

config/bpfd-install/kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+## bpfd CRDs
+- https://github.com/bpfd-dev/bpfd/releases/download/v0.3.0/bpfd-crds-install-v0.3.0.yaml
+## bpfd Operator
+- https://github.com/bpfd-dev/bpfd/releases/download/v0.3.0/bpfd-operator-install-v0.3.0.yaml
+
+patches:
+- path: patch.yaml
+  target:
+    kind: ConfigMap
+    name: config
+    version: v1
+    name: bpfd-config

config/bpfd-install/patch.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config
+data:
+  ## Can be configured at runtime
+  bpfd.log.level: "debug"
+  bpfd.agent.log.level: "debug"
+  bpfd.enable.csi: "true"
+  ## Custom temporary blixt build
+  bpfd.image: quay.io/bpfd/bpfd:blixt

config/bpfd/bpfd.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: dataplane
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: dataplane
+        securityContext:
+          privileged: false
+        volumeMounts:
+        - name: bpf-maps
+          mountPath: /run/bpfd/fs/maps
+          readOnly: true
+      volumes:
+      - name: bpf-maps
+        csi:
+          driver: csi.bpfd.dev
+          volumeAttributes:
+            csi.bpfd.dev/program: blixt-tc-egress
+            csi.bpfd.dev/maps: AYA_LOGS,AYA_LOG_BUF,BACKENDS,BLIXT_CONNTRACK,GATEWAY_INDEXES

config/bpfd/kustomization.yaml

@@ -0,0 +1,50 @@
+# Adds namespace to all resources.
+namespace: blixt-system
+
+# Value of this field is prepended to the
+# names of all resources, e.g. a deployment named
+# "wordpress" becomes "alices-wordpress".
+# Note that it should also match with the prefix (text before '-') of the namespace
+# field above.
+namePrefix: blixt-
+
+# Labels to add to all resources and selectors.
+#commonLabels:
+#  someName: someValue
+
+# - ../crd # TODO: no CRDs yet
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
+#- ../webhook
+# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
+#- ../certmanager
+# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
+#- ../prometheus
+
+# Protect the /metrics endpoint by putting it behind auth.
+# If you want your controlplane to expose the /metrics
+# endpoint w/o any authn/z, please comment the following line.
+
+# Mount the controller config file for loading manager configurations
+# through a ComponentConfig type
+#- manager_config_patch.yaml
+
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
+#- manager_webhook_patch.yaml
+
+# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
+# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
+# 'CERTMANAGER' needs to be enabled to use ca injection
+#- webhookcainjection_patch.yaml
+
+# the following config is for teaching kustomize how to do var substitution
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../bpf-bytecode
+- ../rbac
+- ../manager
+- ../dataplane
+patches:
+- path: bpfd.yaml

config/tests/integration-bpfd/kustomization.yaml

@@ -0,0 +1,12 @@
+
+images:
+- name: ghcr.io/kong/blixt-dataplane
+  newTag: integration-tests
+- name: ghcr.io/kong/blixt-controlplane
+  newTag: integration-tests
+- name: ghcr.io/kong/blixt-udp-test-server
+  newTag: integration-tests
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../bpfd

test/integration/suite_test.go

@@ -24,12 +24,15 @@ import (
 	"fmt"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters"
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/loadimage"
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/metallb"
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters/types/kind"
 	"github.com/kong/kubernetes-testing-framework/pkg/environments"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/gateway-api/pkg/client/clientset/versioned"
 
 	testutils "github.com/kong/blixt/internal/test/utils"
@@ -50,13 +53,20 @@ var (
 	existingCluster      = os.Getenv("BLIXT_USE_EXISTING_KIND_CLUSTER")
 	keepTestCluster      = func() bool { return os.Getenv("BLIXT_TEST_KEEP_CLUSTER") == "true" || existingCluster != "" }()
 	keepKustomizeDeploys = func() bool { return os.Getenv("BLIXT_TEST_KEEP_KUSTOMIZE_DEPLOYS") == "true" }()
+	blixtUseBpfd         = func() bool { return os.Getenv("BLIXT_USE_BPFD") == "true" && existingCluster == "" }()
 
 	mainCleanupKey = "main"
 )
 
 const (
-	gwCRDsKustomize = "https://github.com/kubernetes-sigs/gateway-api/config/crd/experimental?ref=v0.8.1"
-	testKustomize   = "../../config/tests/integration"
+	gwCRDsKustomize          = "https://github.com/kubernetes-sigs/gateway-api/config/crd/experimental?ref=v0.8.1"
+	testKustomizeBpfdInstall = "../../config/bpfd-install"
+	testKustomize            = "../../config/tests/integration"
+	testKustomizeBpfd        = "../../config/tests/integration-bpfd"
+	bpfdNs                   = "bpfd"
+	bpfdConfigName           = "bpfd-config"
+	bpfdOperatorName         = "bpfd-operator"
+	bpfdDsName               = "bpfd-daemon"
 )
 
 func TestMain(m *testing.M) {
@@ -112,6 +122,23 @@ func TestMain(m *testing.M) {
 	gwclient, err = versioned.NewForConfig(env.Cluster().Config())
 	exitOnErr(err)
 
+	if blixtUseBpfd {
+		// deploy bpfd for blixt
+		fmt.Println("INFO: deploying bpfd")
+		exitOnErr(clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testKustomizeBpfdInstall))
+		if !keepKustomizeDeploys {
+			addCleanup(mainCleanupKey, func(context.Context) error {
+				cleanupLog("delete bpfd configmap to cleanup bpfd daemon")
+				env.Cluster().Client().CoreV1().ConfigMaps(bpfdNs).Delete(ctx, bpfdConfigName, metav1.DeleteOptions{})
+				waitForBpfdConfigDelete(ctx, env)
+				cleanupLog("deleting bpfd namespace")
+				return env.Cluster().Client().CoreV1().Namespaces().Delete(ctx, bpfdNs, metav1.DeleteOptions{})
+			})
+		}
+
+		exitOnErr(waitForBpfdReadiness(ctx, env))
+	}
+
 	// deploy the Gateway API CRDs
 	fmt.Println("INFO: deploying Gateway API CRDs")
 	exitOnErr(clusters.KustomizeDeployForCluster(ctx, env.Cluster(), gwCRDsKustomize))
@@ -122,15 +149,28 @@ func TestMain(m *testing.M) {
 		})
 	}
 
-	// deploy the blixt controlplane and dataplane, rbac permissions, e.t.c.
-	// this is what the tests will actually run against.
-	fmt.Println("INFO: deploying blixt via config/test kustomize")
-	exitOnErr(clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testKustomize))
-	if !keepKustomizeDeploys {
-		addCleanup(mainCleanupKey, func(context.Context) error {
-			cleanupLog("cleaning up blixt via config/test kustomize")
-			return clusters.KustomizeDeleteForCluster(ctx, env.Cluster(), testKustomize)
-		})
+	if blixtUseBpfd {
+		// deploy the blixt controlplane and dataplane, rbac permissions, e.t.c.
+		// this is what the tests will actually run against.
+		fmt.Println("INFO: deploying blixt via config/test-bpfd kustomize")
+		exitOnErr(clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testKustomizeBpfd))
+		if !keepKustomizeDeploys {
+			addCleanup(mainCleanupKey, func(context.Context) error {
+				cleanupLog("cleaning up blixt via config/test kustomize")
+				return clusters.KustomizeDeleteForCluster(ctx, env.Cluster(), testKustomizeBpfd)
+			})
+		}
+	} else {
+		// deploy the blixt controlplane and dataplane, rbac permissions, e.t.c.
+		// this is what the tests will actually run against.
+		fmt.Println("INFO: deploying blixt via config/test kustomize")
+		exitOnErr(clusters.KustomizeDeployForCluster(ctx, env.Cluster(), testKustomize))
+		if !keepKustomizeDeploys {
+			addCleanup(mainCleanupKey, func(context.Context) error {
+				cleanupLog("cleaning up blixt via config/test kustomize")
+				return clusters.KustomizeDeleteForCluster(ctx, env.Cluster(), testKustomize)
+			})
+		}
 	}
 
 	fmt.Println("INFO: waiting for Blixt component readiness")
@@ -195,3 +235,72 @@ func runCleanup(cleanupKey string) (cleanupErr error) {
 	delete(cleanup, cleanupKey)
 	return
 }
+
+func waitForBpfdReadiness(ctx context.Context, env environments.Environment) error {
+	for {
+		time.Sleep(2 * time.Second)
+		select {
+		case <-ctx.Done():
+			if err := ctx.Err(); err != nil {
+				return fmt.Errorf("context completed while waiting for components: %w", err)
+			}
+			return fmt.Errorf("context completed while waiting for components")
+		default:
+			fmt.Println("INFO: waiting for bpfd")
+			var controlplaneReady, dataplaneReady bool
+
+			controlplane, err := env.Cluster().Client().AppsV1().Deployments(bpfdNs).Get(ctx, bpfdOperatorName, metav1.GetOptions{})
+			if err != nil {
+				if errors.IsNotFound(err) {
+					fmt.Println("INFO: bpfd-operator dep not found yet")
+					continue
+				}
+				return err
+			}
+			if controlplane.Status.AvailableReplicas > 0 {
+				controlplaneReady = true
+			}
+
+			dataplane, err := env.Cluster().Client().AppsV1().DaemonSets(bpfdNs).Get(ctx, bpfdDsName, metav1.GetOptions{})
+			if err != nil {
+				if errors.IsNotFound(err) {
+					fmt.Println("INFO: bpfd daemon not found yet")
+					continue
+				}
+				return err
+			}
+			if dataplane.Status.NumberAvailable > 0 {
+				dataplaneReady = true
+			}
+
+			if controlplaneReady && dataplaneReady {
+				fmt.Println("INFO: bpfd-operator is ready")
+				return nil
+			}
+		}
+	}
+}
+
+func waitForBpfdConfigDelete(ctx context.Context, env environments.Environment) error {
+	for {
+		time.Sleep(2 * time.Second)
+		select {
+		case <-ctx.Done():
+			if err := ctx.Err(); err != nil {
+				return fmt.Errorf("context completed while waiting for components: %w", err)
+			}
+			return fmt.Errorf("context completed while waiting for components")
+		default:
+			fmt.Println("INFO: waiting for bpfd config deletion")
+
+			_, err := env.Cluster().Client().CoreV1().ConfigMaps(bpfdNs).Get(ctx, bpfdConfigName, metav1.GetOptions{})
+			if err != nil {
+				if errors.IsNotFound(err) {
+					fmt.Println("INFO: bpfd configmap deleted successfully")
+					return nil
+				}
+				return err
+			}
+		}
+	}
+}