From 2719f2eafcd4055417ed255d869fa2746eb8ff27 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Mon, 16 Oct 2023 13:50:46 +0300 Subject: [PATCH] plugins: add sgx-config plugin. Add a sample plugin which perform SGX EPC resource configuration based on pod annotations. Note that this plugin needs a patched kernel with support for something like echo 'sgx_epc 65536' > /sys/fs/cgroup/$CGRP/misc.max to work. Signed-off-by: Krisztian Litkey --- Makefile | 3 +- cmd/plugins/sgx-config/Dockerfile | 22 +++ .../nri-sgx-config-deployment.yaml.in | 38 ++++ cmd/plugins/sgx-config/sgx-config.go | 184 ++++++++++++++++++ deployment/helm/sgx-config/Chart.yaml | 11 ++ .../helm/sgx-config/templates/_helpers.tpl | 16 ++ .../helm/sgx-config/templates/daemonset.yaml | 68 +++++++ deployment/helm/sgx-config/values.scheme.json | 44 +++++ deployment/helm/sgx-config/values.yaml | 22 +++ deployment/overlays/sgx-config/daemonset.yaml | 38 ++++ .../overlays/sgx-config/kustomization.yaml | 12 ++ docs/memory/sgx-config.md | 27 +++ .../files/nri-sgx-config-deployment.yaml.in | 38 ++++ 13 files changed, 522 insertions(+), 1 deletion(-) create mode 100644 cmd/plugins/sgx-config/Dockerfile create mode 100644 cmd/plugins/sgx-config/nri-sgx-config-deployment.yaml.in create mode 100644 cmd/plugins/sgx-config/sgx-config.go create mode 100644 deployment/helm/sgx-config/Chart.yaml create mode 100644 deployment/helm/sgx-config/templates/_helpers.tpl create mode 100644 deployment/helm/sgx-config/templates/daemonset.yaml create mode 100644 deployment/helm/sgx-config/values.scheme.json create mode 100644 deployment/helm/sgx-config/values.yaml create mode 100644 deployment/overlays/sgx-config/daemonset.yaml create mode 100644 deployment/overlays/sgx-config/kustomization.yaml create mode 100644 docs/memory/sgx-config.md create mode 100644 test/e2e/files/nri-sgx-config-deployment.yaml.in diff --git a/Makefile b/Makefile index d847ba981..f790959ab 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,8 @@ PLUGINS ?= \ nri-resource-policy-balloons \ nri-resource-policy-template \ nri-memory-qos \ - nri-memtierd + nri-memtierd \ + nri-sgx-config BINARIES ?= \ config-manager diff --git a/cmd/plugins/sgx-config/Dockerfile b/cmd/plugins/sgx-config/Dockerfile new file mode 100644 index 000000000..dabe20897 --- /dev/null +++ b/cmd/plugins/sgx-config/Dockerfile @@ -0,0 +1,22 @@ +ARG GO_VERSION=1.20 + +FROM golang:${GO_VERSION}-bullseye as builder + +WORKDIR /go/builder + +# Fetch go dependencies in a separate layer for caching +COPY go.mod go.sum ./ +COPY pkg/topology/ pkg/topology/ +RUN go mod download + +# Build nri-resmgr +COPY . . + +RUN make clean +RUN make PLUGINS=nri-sgx-config build-plugins-static + +FROM gcr.io/distroless/static + +COPY --from=builder /go/builder/build/bin/nri-sgx-config /bin/nri-sgx-config + +ENTRYPOINT ["/bin/nri-sgx-config", "-idx", "40"] diff --git a/cmd/plugins/sgx-config/nri-sgx-config-deployment.yaml.in b/cmd/plugins/sgx-config/nri-sgx-config-deployment.yaml.in new file mode 100644 index 000000000..aba07c02b --- /dev/null +++ b/cmd/plugins/sgx-config/nri-sgx-config-deployment.yaml.in @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: nri-sgx-config + name: nri-sgx-config + namespace: kube-system +spec: + selector: + matchLabels: + app: nri-sgx-config + template: + metadata: + labels: + app: nri-sgx-config + spec: + nodeSelector: + kubernetes.io/os: "linux" + containers: + - name: nri-sgx-config + command: + - nri-sgx-config + - --idx + - "40" + image: IMAGE_PLACEHOLDER + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: nri-sockets-vol + mountPath: /var/run/nri + volumes: + - name: nri-sockets-vol + hostPath: + path: /var/run/nri + type: Directory diff --git a/cmd/plugins/sgx-config/sgx-config.go b/cmd/plugins/sgx-config/sgx-config.go new file mode 100644 index 000000000..83fc876dd --- /dev/null +++ b/cmd/plugins/sgx-config/sgx-config.go @@ -0,0 +1,184 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "fmt" + "os" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + "sigs.k8s.io/yaml" + + "github.com/containerd/nri/pkg/api" + "github.com/containerd/nri/pkg/stub" +) + +const ( + // Key used for SGX annotations. + sgxEpcKey = "sgx-epc.nri.io" +) + +var ( + log *logrus.Logger + verbose bool +) + +// our injector plugin +type plugin struct { + stub stub.Stub +} + +// CreateContainer handles container creation requests. +func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, container *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) { + var ( + ctrName string + sgxEpc uint64 + err error + ) + + ctrName = containerName(pod, container) + + if verbose { + dump("CreateContainer", "pod", pod, "container", container) + } else { + log.Infof("CreateContainer %s", ctrName) + } + + adjust := &api.ContainerAdjustment{} + + sgxEpc, err = parseSgxEpc(container.Name, pod.Annotations) + if err != nil { + log.Errorf("failed to parse SGX EPC annotation: %v", err) + return nil, nil, err + } + + if sgxEpc > 0 { + adjust.AddLinuxUnified("misc.max", "sgx_epc "+strconv.FormatUint(sgxEpc, 10)) + + if verbose { + dump(ctrName, "ContainerAdjustment", adjust) + } else { + log.Infof("adjusted SGX EPC to %d", sgxEpc) + } + } else { + log.Infof("no SGX EPC annotations") + } + + return adjust, nil, nil +} + +func parseSgxEpc(ctr string, annotations map[string]string) (uint64, error) { + // check container-specific or pod-global SGX EPC annotation and parse it + for _, key := range []string{ + sgxEpcKey + "/container." + ctr, + sgxEpcKey + "/pod", + sgxEpcKey, + } { + if value, ok := annotations[key]; ok { + epc, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse annotation %s: %w", value, err) + } + return epc, nil + } + } + + return 0, nil +} + +// Construct a container name for log messages. +func containerName(pod *api.PodSandbox, container *api.Container) string { + if pod != nil { + return pod.Name + "/" + container.Name + } + return container.Name +} + +// Dump one or more objects, with an optional global prefix and per-object tags. +func dump(args ...interface{}) { + var ( + prefix string + idx int + ) + + if len(args)&0x1 == 1 { + prefix = args[0].(string) + idx++ + } + + for ; idx < len(args)-1; idx += 2 { + tag, obj := args[idx], args[idx+1] + msg, err := yaml.Marshal(obj) + if err != nil { + log.Infof("%s: %s: failed to dump object: %v", prefix, tag, err) + continue + } + + if prefix != "" { + log.Infof("%s: %s:", prefix, tag) + for _, line := range strings.Split(strings.TrimSpace(string(msg)), "\n") { + log.Infof("%s: %s", prefix, line) + } + } else { + log.Infof("%s:", tag) + for _, line := range strings.Split(strings.TrimSpace(string(msg)), "\n") { + log.Infof(" %s", line) + } + } + } +} + +func main() { + var ( + pluginName string + pluginIdx string + opts []stub.Option + err error + ) + + log = logrus.StandardLogger() + log.SetFormatter(&logrus.TextFormatter{ + PadLevelText: true, + }) + + flag.StringVar(&pluginName, "name", "", "plugin name to register to NRI") + flag.StringVar(&pluginIdx, "idx", "", "plugin index to register to NRI") + flag.BoolVar(&verbose, "verbose", false, "enable (more) verbose logging") + flag.Parse() + + if pluginName != "" { + opts = append(opts, stub.WithPluginName(pluginName)) + } + if pluginIdx != "" { + opts = append(opts, stub.WithPluginIdx(pluginIdx)) + } + + p := &plugin{} + if p.stub, err = stub.New(p, opts...); err != nil { + log.Fatalf("failed to create plugin stub: %v", err) + } + + err = p.stub.Run(context.Background()) + if err != nil { + log.Errorf("plugin exited with error %v", err) + os.Exit(1) + } +} diff --git a/deployment/helm/sgx-config/Chart.yaml b/deployment/helm/sgx-config/Chart.yaml new file mode 100644 index 000000000..ac350ee1b --- /dev/null +++ b/deployment/helm/sgx-config/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +appVersion: unstable +description: | + The sgx-config NRI plugin allows control over SGX resource usage using the + cgroup v2 misc controller and pod annotations. +name: nri-sgx-config +sources: + - https://github.com/containers/nri-plugins +home: https://github.com/containers/nri-plugins +type: application +version: v0.0.0 diff --git a/deployment/helm/sgx-config/templates/_helpers.tpl b/deployment/helm/sgx-config/templates/_helpers.tpl new file mode 100644 index 000000000..43211338c --- /dev/null +++ b/deployment/helm/sgx-config/templates/_helpers.tpl @@ -0,0 +1,16 @@ +{{/* +Common labels +*/}} +{{- define "sgx-config.labels" -}} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{ include "sgx-config.selectorLabels" . }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "sgx-config.selectorLabels" -}} +app.kubernetes.io/name: nri-sgx-config +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/deployment/helm/sgx-config/templates/daemonset.yaml b/deployment/helm/sgx-config/templates/daemonset.yaml new file mode 100644 index 000000000..0b7a007ce --- /dev/null +++ b/deployment/helm/sgx-config/templates/daemonset.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + {{- include "sgx-config.labels" . | nindent 4 }} + name: nri-sgx-config + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "sgx-config.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "sgx-config.labels" . | nindent 8 }} + spec: + restartPolicy: OnFailure + nodeSelector: + kubernetes.io/os: "linux" + {{- if .Values.nri.patchRuntimeConfig }} + initContainers: + - name: patch-runtime + image: {{ .Values.initContainerImage.name }}:{{ .Values.initContainerImage.tag | default .Chart.AppVersion }} + imagePullPolicy: {{ .Values.initContainerImage.pullPolicy }} + volumeMounts: + - name: containerd-config + mountPath: /etc/containerd + - name: crio-config + mountPath: /etc/crio/crio.conf.d + - name: dbus-socket + mountPath: /var/run/dbus/system_bus_socket + securityContext: + privileged: true + {{- end }} + containers: + - name: nri-sgx-config + command: + - nri-sgx-config + - --idx + - "40" + image: {{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + requests: + cpu: {{ .Values.resources.cpu }} + memory: {{ .Values.resources.memory }} + volumeMounts: + - name: nrisockets + mountPath: /var/run/nri + volumes: + - name: nrisockets + hostPath: + path: /var/run/nri + type: Directory + {{- if .Values.nri.patchRuntimeConfig }} + - name: containerd-config + hostPath: + path: /etc/containerd/ + type: DirectoryOrCreate + - name: crio-config + hostPath: + path: /etc/crio/crio.conf.d/ + type: DirectoryOrCreate + - name: dbus-socket + hostPath: + path: /var/run/dbus/system_bus_socket + type: Socket + {{- end }} diff --git a/deployment/helm/sgx-config/values.scheme.json b/deployment/helm/sgx-config/values.scheme.json new file mode 100644 index 000000000..dbe88c447 --- /dev/null +++ b/deployment/helm/sgx-config/values.scheme.json @@ -0,0 +1,44 @@ +{ + "$schema": "http://json-schema.org/schema#", + "required": [ + "image", + "resources" + ], + "properties": { + "image": { + "type": "object", + "required": [ + "name", + "tag", + "pullPolicy" + ], + "properties": { + "name": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "pullPolicy": { + "type": "string", + "enum": ["Never", "Always", "IfNotPresent"] + } + } + }, + "resources": { + "type": "object", + "required": [ + "cpu", + "memory" + ], + "properties": { + "cpu": { + "type": "integer" + }, + "memory": { + "type": "integer" + } + } + } + } + } diff --git a/deployment/helm/sgx-config/values.yaml b/deployment/helm/sgx-config/values.yaml new file mode 100644 index 000000000..bcc6b7e84 --- /dev/null +++ b/deployment/helm/sgx-config/values.yaml @@ -0,0 +1,22 @@ +# Default values for sgx-config. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +--- +image: + name: ghcr.io/containers/nri-plugins/nri-sgx-config + # tag, if defined will use the given image tag, otherwise Chart.AppVersion will be used + #tag: unstable + pullPolicy: Always + +resources: + cpu: 50m + memory: 100Mi + +nri: + patchRuntimeConfig: false + +initContainerImage: + name: ghcr.io/containers/nri-plugins/nri-config-manager + # If not defined Chart.AppVersion will be used + #tag: unstable + pullPolicy: Always diff --git a/deployment/overlays/sgx-config/daemonset.yaml b/deployment/overlays/sgx-config/daemonset.yaml new file mode 100644 index 000000000..bd03b1c6a --- /dev/null +++ b/deployment/overlays/sgx-config/daemonset.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: nri-sgx-config + name: nri-sgx-config + namespace: kube-system +spec: + selector: + matchLabels: + app: nri-sgx-config + template: + metadata: + labels: + app: nri-sgx-config + spec: + nodeSelector: + kubernetes.io/os: "linux" + containers: + - name: nri-sgx-config + command: + - nri-sgx-config + - --idx + - "40" + image: IMAGE_PLACEHOLDER + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 50m + memory: 100Mi + volumeMounts: + - name: nri-sockets-vol + mountPath: /var/run/nri + volumes: + - name: nri-sockets-vol + hostPath: + path: /var/run/nri + type: Directory diff --git a/deployment/overlays/sgx-config/kustomization.yaml b/deployment/overlays/sgx-config/kustomization.yaml new file mode 100644 index 000000000..b5f1eb450 --- /dev/null +++ b/deployment/overlays/sgx-config/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: kube-system + +images: +- name: '*' + newName: ghcr.io/containers/nri-plugins/nri-sgx-config + newTag: unstable + +resources: +- daemonset.yaml diff --git a/docs/memory/sgx-config.md b/docs/memory/sgx-config.md new file mode 100644 index 000000000..08f9def03 --- /dev/null +++ b/docs/memory/sgx-config.md @@ -0,0 +1,27 @@ +## SGX Resource Configurator Plugin + +This sample plugin can configure SGX resources using annotations. + +### SGX Annotations + +You can annotate SGX EPC configuration for every container in the pod, +or just a specified container using the following annotation notations: +``` +... +metadata: + annotations: + # for all containers in the pod + sgx-epc.nri.io/pod: "32768" + # alternative notation for all containers in the pod + sgx-epc.nri.io: "8192" + # for container c0 in the pod + sgx-epc.nri.io/container.c0: "16384" +... +``` + +## Testing + +You can test this plugin using a kubernetes cluster/node with a container +runtime that has NRI support enabled. Start the plugin on the target node +(`sgx-config -idx 10`), create a pod with some SGX annotation, then verify +that the annotations SGC resource configuration gets properly set. diff --git a/test/e2e/files/nri-sgx-config-deployment.yaml.in b/test/e2e/files/nri-sgx-config-deployment.yaml.in new file mode 100644 index 000000000..aba07c02b --- /dev/null +++ b/test/e2e/files/nri-sgx-config-deployment.yaml.in @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: nri-sgx-config + name: nri-sgx-config + namespace: kube-system +spec: + selector: + matchLabels: + app: nri-sgx-config + template: + metadata: + labels: + app: nri-sgx-config + spec: + nodeSelector: + kubernetes.io/os: "linux" + containers: + - name: nri-sgx-config + command: + - nri-sgx-config + - --idx + - "40" + image: IMAGE_PLACEHOLDER + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: nri-sockets-vol + mountPath: /var/run/nri + volumes: + - name: nri-sockets-vol + hostPath: + path: /var/run/nri + type: Directory