Skip to content

Commit

Permalink
plugins: add sgx-config plugin.
Browse files Browse the repository at this point in the history
Add a sample plugin for SGX EPC resource configuration using
pod annotations. Note that this plugin needs a patched kernel
with support for something like
    echo 'sgx_epc 65536' > /sys/fs/cgroup/$CGRP/misc.max
to work.

Signed-off-by: Krisztian Litkey <[email protected]>
  • Loading branch information
klihub committed Oct 16, 2023
1 parent 2a7193a commit 25a3dbc
Show file tree
Hide file tree
Showing 14 changed files with 516 additions and 1 deletion.
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ PLUGINS ?= \
nri-resource-policy-balloons \
nri-resource-policy-template \
nri-memory-qos \
nri-memtierd
nri-memtierd \
nri-sgx-config

BINARIES ?= \
config-manager
Expand Down
22 changes: 22 additions & 0 deletions cmd/plugins/sgx-config/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
ARG GO_VERSION=1.20

FROM golang:${GO_VERSION}-bullseye as builder

WORKDIR /go/builder

# Fetch go dependencies in a separate layer for caching
COPY go.mod go.sum ./
COPY pkg/topology/ pkg/topology/
RUN go mod download

# Build nri-resmgr
COPY . .

RUN make clean
RUN make PLUGINS=nri-sgx-config build-plugins-static

FROM gcr.io/distroless/static

COPY --from=builder /go/builder/build/bin/nri-sgx-config /bin/nri-sgx-config

ENTRYPOINT ["/bin/nri-sgx-config", "-idx", "40"]
38 changes: 38 additions & 0 deletions cmd/plugins/sgx-config/nri-sgx-config-deployment.yaml.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
app: nri-sgx-config
name: nri-sgx-config
namespace: kube-system
spec:
selector:
matchLabels:
app: nri-sgx-config
template:
metadata:
labels:
app: nri-sgx-config
spec:
nodeSelector:
kubernetes.io/os: "linux"
containers:
- name: nri-sgx-config
command:
- nri-sgx-config
- --idx
- "40"
image: IMAGE_PLACEHOLDER
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: nri-sockets-vol
mountPath: /var/run/nri
volumes:
- name: nri-sockets-vol
hostPath:
path: /var/run/nri
type: Directory
184 changes: 184 additions & 0 deletions cmd/plugins/sgx-config/sgx-config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"context"
"flag"
"fmt"
"os"
"strconv"
"strings"

"github.com/sirupsen/logrus"
"sigs.k8s.io/yaml"

"github.com/containerd/nri/pkg/api"
"github.com/containerd/nri/pkg/stub"
)

const (
// Key used for SGX annotations.
sgxEpcKey = "sgx-epc.nri.io"
)

var (
log *logrus.Logger
verbose bool
)

// our injector plugin
type plugin struct {
stub stub.Stub
}

// CreateContainer handles container creation requests.
func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, container *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) {
var (
ctrName string
sgxEpc uint64
err error
)

ctrName = containerName(pod, container)

if verbose {
dump("CreateContainer", "pod", pod, "container", container)
} else {
log.Infof("CreateContainer %s", ctrName)
}

adjust := &api.ContainerAdjustment{}

sgxEpc, err = parseSgxEpc(container.Name, pod.Annotations)
if err != nil {
log.Errorf("failed to parse SGX EPC annotation: %v", err)
return nil, nil, err
}

if sgxEpc > 0 {
adjust.AddLinuxUnified("misc.max", "sgx_epc "+strconv.FormatUint(sgxEpc, 10))

if verbose {
dump(ctrName, "ContainerAdjustment", adjust)
} else {
log.Infof("adjusted SGX EPC to %d", sgxEpc)
}
} else {
log.Infof("no SGX EPC annotations")
}

return adjust, nil, nil
}

func parseSgxEpc(ctr string, annotations map[string]string) (uint64, error) {
// check container-specific or pod-global SGX EPC annotation and parse it
for _, key := range []string{
sgxEpcKey + "/container." + ctr,
sgxEpcKey + "/pod",
sgxEpcKey,
} {
if value, ok := annotations[key]; ok {
epc, err := strconv.ParseUint(value, 10, 64)
if err != nil {
return 0, fmt.Errorf("failed to parse annotation %s: %w", value, err)
}
return epc, nil
}
}

return 0, nil
}

// Construct a container name for log messages.
func containerName(pod *api.PodSandbox, container *api.Container) string {
if pod != nil {
return pod.Name + "/" + container.Name
}
return container.Name
}

// Dump one or more objects, with an optional global prefix and per-object tags.
func dump(args ...interface{}) {
var (
prefix string
idx int
)

if len(args)&0x1 == 1 {
prefix = args[0].(string)
idx++
}

for ; idx < len(args)-1; idx += 2 {
tag, obj := args[idx], args[idx+1]
msg, err := yaml.Marshal(obj)
if err != nil {
log.Infof("%s: %s: failed to dump object: %v", prefix, tag, err)
continue
}

if prefix != "" {
log.Infof("%s: %s:", prefix, tag)
for _, line := range strings.Split(strings.TrimSpace(string(msg)), "\n") {
log.Infof("%s: %s", prefix, line)
}
} else {
log.Infof("%s:", tag)
for _, line := range strings.Split(strings.TrimSpace(string(msg)), "\n") {
log.Infof(" %s", line)
}
}
}
}

func main() {
var (
pluginName string
pluginIdx string
opts []stub.Option
err error
)

log = logrus.StandardLogger()
log.SetFormatter(&logrus.TextFormatter{
PadLevelText: true,
})

flag.StringVar(&pluginName, "name", "", "plugin name to register to NRI")
flag.StringVar(&pluginIdx, "idx", "", "plugin index to register to NRI")
flag.BoolVar(&verbose, "verbose", false, "enable (more) verbose logging")
flag.Parse()

if pluginName != "" {
opts = append(opts, stub.WithPluginName(pluginName))
}
if pluginIdx != "" {
opts = append(opts, stub.WithPluginIdx(pluginIdx))
}

p := &plugin{}
if p.stub, err = stub.New(p, opts...); err != nil {
log.Fatalf("failed to create plugin stub: %v", err)
}

err = p.stub.Run(context.Background())
if err != nil {
log.Errorf("plugin exited with error %v", err)
os.Exit(1)
}
}
11 changes: 11 additions & 0 deletions deployment/helm/sgx-config/Chart.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
apiVersion: v2
appVersion: unstable
description: |
The sgx-config NRI plugin allows control over SGX resource usage using the
cgroup v2 misc controller and pod annotations.
name: nri-sgx-config
sources:
- https://github.com/containers/nri-plugins
home: https://github.com/containers/nri-plugins
type: application
version: v0.0.0
16 changes: 16 additions & 0 deletions deployment/helm/sgx-config/templates/_helpers.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{{/*
Common labels
*/}}
{{- define "sgx-config.labels" -}}
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{ include "sgx-config.selectorLabels" . }}
{{- end -}}

{{/*
Selector labels
*/}}
{{- define "sgx-config.selectorLabels" -}}
app.kubernetes.io/name: nri-sgx-config
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end -}}
68 changes: 68 additions & 0 deletions deployment/helm/sgx-config/templates/daemonset.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
{{- include "sgx-config.labels" . | nindent 4 }}
name: nri-sgx-config
namespace: {{ .Release.Namespace }}
spec:
selector:
matchLabels:
{{- include "sgx-config.selectorLabels" . | nindent 6 }}
template:
metadata:
labels:
{{- include "sgx-config.labels" . | nindent 8 }}
spec:
restartPolicy: OnFailure
nodeSelector:
kubernetes.io/os: "linux"
{{- if .Values.nri.patchRuntimeConfig }}
initContainers:
- name: patch-runtime
image: {{ .Values.initContainerImage.name }}:{{ .Values.initContainerImage.tag | default .Chart.AppVersion }}
imagePullPolicy: {{ .Values.initContainerImage.pullPolicy }}
volumeMounts:
- name: containerd-config
mountPath: /etc/containerd
- name: crio-config
mountPath: /etc/crio/crio.conf.d
- name: dbus-socket
mountPath: /var/run/dbus/system_bus_socket
securityContext:
privileged: true
{{- end }}
containers:
- name: nri-sgx-config
command:
- nri-sgx-config
- --idx
- "40"
image: {{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
resources:
requests:
cpu: {{ .Values.resources.cpu }}
memory: {{ .Values.resources.memory }}
volumeMounts:
- name: nrisockets
mountPath: /var/run/nri
volumes:
- name: nrisockets
hostPath:
path: /var/run/nri
type: Directory
{{- if .Values.nri.patchRuntimeConfig }}
- name: containerd-config
hostPath:
path: /etc/containerd/
type: DirectoryOrCreate
- name: crio-config
hostPath:
path: /etc/crio/crio.conf.d/
type: DirectoryOrCreate
- name: dbus-socket
hostPath:
path: /var/run/dbus/system_bus_socket
type: Socket
{{- end }}
44 changes: 44 additions & 0 deletions deployment/helm/sgx-config/values.scheme.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"$schema": "http://json-schema.org/schema#",
"required": [
"image",
"resources"
],
"properties": {
"image": {
"type": "object",
"required": [
"name",
"tag",
"pullPolicy"
],
"properties": {
"name": {
"type": "string"
},
"tag": {
"type": "string"
},
"pullPolicy": {
"type": "string",
"enum": ["Never", "Always", "IfNotPresent"]
}
}
},
"resources": {
"type": "object",
"required": [
"cpu",
"memory"
],
"properties": {
"cpu": {
"type": "integer"
},
"memory": {
"type": "integer"
}
}
}
}
}
Loading

0 comments on commit 25a3dbc

Please sign in to comment.