From 43e5f33a3d1ef73a44634905c4053c7309d162bc Mon Sep 17 00:00:00 2001 From: anjmao Date: Thu, 7 Dec 2023 16:12:43 +0200 Subject: [PATCH] Add only needed capabilities --- .../templates/collector/daemonSet.yaml | 27 +++++++++++++---- charts/egressd/values-tilt.yaml | 3 ++ charts/egressd/values.yaml | 9 +++++- cmd/collector/main.go | 29 +++++++++++++++++++ conntrack/nf_conntrack.go | 6 +--- ebpf/tracer_linux.go | 10 +++++++ ebpf/tracer_nolinux.go | 4 +++ go.mod | 2 +- go.sum | 2 ++ 9 files changed, 79 insertions(+), 13 deletions(-) diff --git a/charts/egressd/templates/collector/daemonSet.yaml b/charts/egressd/templates/collector/daemonSet.yaml index 17507d6..d1ee07a 100644 --- a/charts/egressd/templates/collector/daemonSet.yaml +++ b/charts/egressd/templates/collector/daemonSet.yaml @@ -37,6 +37,25 @@ spec: hostNetwork: true securityContext: {{- toYaml .Values.collector.podSecurityContext | nindent 8 }} + initContainers: + - name: egressd-init + securityContext: + privileged: true + image: "{{ .Values.collector.image.repository }}:{{ .Values.collector.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.collector.image.pullPolicy }} + args: + - "-init" + volumeMounts: + - mountPath: /hostproc/sys/net/netfilter + name: netfilter + mountPropagation: Bidirectional + {{- if (get .Values.collector.extraArgs "ebpf-dns-tracer-enabled") }} + - mountPath: /sys/fs/cgroup + name: cgroup + mountPropagation: Bidirectional + - mountPath: /cgroup2-manual-mount + name: cgroup-mountdir + {{- end }} containers: - name: egressd securityContext: @@ -61,17 +80,13 @@ spec: - mountPath: /sys/fs/bpf mountPropagation: HostToContainer name: cilium-bpf-maps - - mountPath: /var/run/egressd - name: export-logs - mountPath: /hostproc/sys/net/netfilter name: netfilter - mountPropagation: Bidirectional + readOnly: true {{- if (get .Values.collector.extraArgs "ebpf-dns-tracer-enabled") }} - mountPath: /sys/fs/cgroup name: cgroup - mountPropagation: Bidirectional - - mountPath: /cgroup2-manual-mount - name: cgroup-mountdir + readOnly: true {{- end }} ports: - containerPort: {{.Values.collector.httpListenPort}} diff --git a/charts/egressd/values-tilt.yaml b/charts/egressd/values-tilt.yaml index 5be5569..f44ee53 100644 --- a/charts/egressd/values-tilt.yaml +++ b/charts/egressd/values-tilt.yaml @@ -24,3 +24,6 @@ exporter: tilt_prom: prom_remote_write: url: "http://victoria-metrics-single-server:8428/api/v1/write" + +castai: + apiKey: "test" \ No newline at end of file diff --git a/charts/egressd/values.yaml b/charts/egressd/values.yaml index ec08ffb..c673b92 100644 --- a/charts/egressd/values.yaml +++ b/charts/egressd/values.yaml @@ -53,8 +53,15 @@ collector: # fsGroup: 2000 containerSecurityContext: - privileged: true + privileged: false readOnlyRootFilesystem: true + capabilities: + drop: + - all + add: + - "NET_ADMIN" # Allows managing network devices and configuring network interfaces. + - "SYS_PTRACE" # Allows ptrace() of any process. + - "SYS_ADMIN" # Allows a wide range of system administration tasks, such as mounting/unmounting filesystems, setting the domainname and hostname, and configuring process accounting and resource limits. resources: requests: diff --git a/cmd/collector/main.go b/cmd/collector/main.go index 7fa61ac..a00b36d 100644 --- a/cmd/collector/main.go +++ b/cmd/collector/main.go @@ -43,6 +43,9 @@ var ( sendTrafficDelta = flag.Bool("send-traffic-delta", false, "Send traffic delta between reads of conntrack entry. Traffic counter is sent by default") ebpfDNSTracerEnabled = flag.Bool("ebpf-dns-tracer-enabled", true, "Enable DNS tracer using eBPF") ebpfDNSTracerQueueSize = flag.Int("ebpf-dns-tracer-queue-size", 1000, "Size of the queue for DNS tracer") + // Kubernetes requires container to run in privileged mode if Bidirectional mount is used. + // Actually it needs only SYS_ADMIN but see this issue See https://github.com/kubernetes/kubernetes/pull/117812 + initMode = flag.Bool("init", false, "Run in init mode to setup conntrack accounting and mount cgroup v2") ) // These should be set via `go build` during a release. @@ -69,6 +72,13 @@ func main() { return } + if *initMode { + if err := runInit(log); err != nil { + log.Fatal(err) + } + return + } + if err := run(log); err != nil && !errors.Is(err, context.Canceled) { log.Fatal(err) } @@ -194,6 +204,25 @@ func run(log logrus.FieldLogger) error { return <-errc } +// runInit runs once in init container. +func runInit(log logrus.FieldLogger) error { + log.Infof("egressd: running init") + defer log.Infof("egressd: init done") + + ciliumAvailable := conntrack.CiliumAvailable() + if !ciliumAvailable { + if err := conntrack.InitNetfilterAccounting(); err != nil { + return fmt.Errorf("init nf conntrack: %w", err) + } + } + + if err := ebpf.InitCgroupv2(); err != nil { + return fmt.Errorf("init cgroupv2: %w", err) + } + + return nil +} + func retrieveKubeConfig(log logrus.FieldLogger, kubepath string) (*rest.Config, error) { if kubepath != "" { data, err := os.ReadFile(kubepath) diff --git a/conntrack/nf_conntrack.go b/conntrack/nf_conntrack.go index 03cf6a2..45e8c0c 100644 --- a/conntrack/nf_conntrack.go +++ b/conntrack/nf_conntrack.go @@ -22,15 +22,11 @@ var ( accounting = "/hostproc/sys/net/netfilter/nf_conntrack_acct" ) -func initNetfilterAccounting() error { +func InitNetfilterAccounting() error { return os.WriteFile(accounting, []byte{'1'}, 0600) } func NewNetfilterClient(log logrus.FieldLogger) (Client, error) { - err := initNetfilterAccounting() - if err != nil { - return nil, fmt.Errorf("initing nf_conntrack_acct: %w", err) - } nfct, err := ct.Open(&ct.Config{}) if err != nil { return nil, fmt.Errorf("opening nfct: %w", err) diff --git a/ebpf/tracer_linux.go b/ebpf/tracer_linux.go index 83c946c..e86cda1 100644 --- a/ebpf/tracer_linux.go +++ b/ebpf/tracer_linux.go @@ -124,6 +124,16 @@ func IsKernelBTFAvailable() bool { return err == nil } +func InitCgroupv2() error { + _, err := detectCgroupPath() + if errors.Is(err, ErrCgroup2NotMounted) { + if err := mountCgroup2(); err != nil { + return fmt.Errorf("cgroup2 not mounted and failed to mount manually: %w", err) + } + } + return nil +} + func parseEvent(data []byte) (DNSEvent, error) { packet := gopacket.NewPacket( data, diff --git a/ebpf/tracer_nolinux.go b/ebpf/tracer_nolinux.go index 046b9cb..fdfa2fc 100644 --- a/ebpf/tracer_nolinux.go +++ b/ebpf/tracer_nolinux.go @@ -17,3 +17,7 @@ func (t *Tracer) Events() <-chan DNSEvent { func IsKernelBTFAvailable() bool { return false } + +func InitCgroupv2() error { + return nil +} diff --git a/go.mod b/go.mod index bc8e10d..affa827 100644 --- a/go.mod +++ b/go.mod @@ -97,7 +97,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect go.mongodb.org/mongo-driver v1.10.2 // indirect go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect - go4.org/unsafe/assume-no-moving-gc v0.0.0-20230209150437-ee73d164e760 // indirect + go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 // indirect golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.1.0 // indirect diff --git a/go.sum b/go.sum index 0e1b69c..6839d91 100644 --- a/go.sum +++ b/go.sum @@ -502,6 +502,8 @@ go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1: go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= go4.org/unsafe/assume-no-moving-gc v0.0.0-20230209150437-ee73d164e760 h1:gH0IO5GDYAcawu+ThKrvAofVTgJjYaoOZ5rrC4pS2Xw= go4.org/unsafe/assume-no-moving-gc v0.0.0-20230209150437-ee73d164e760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 h1:lGdhQUN/cnWdSH3291CUuxSEqc+AsGTiDxPP3r2J0l4= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=