Add only needed capabilities

charts/egressd/templates/collector/daemonSet.yaml

@@ -37,6 +37,25 @@ spec:
       hostNetwork: true
       securityContext:
         {{- toYaml .Values.collector.podSecurityContext | nindent 8 }}
+      initContainers:
+        - name: egressd-init
+          securityContext:
+            privileged: true
+          image: "{{ .Values.collector.image.repository }}:{{ .Values.collector.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.collector.image.pullPolicy }}
+          args:
+            - "-init"
+          volumeMounts:
+            - mountPath: /hostproc/sys/net/netfilter
+              name: netfilter
+              mountPropagation: Bidirectional
+          {{- if (get .Values.collector.extraArgs "ebpf-dns-tracer-enabled") }}
+            - mountPath: /sys/fs/cgroup
+              name: cgroup
+              mountPropagation: Bidirectional
+            - mountPath: /cgroup2-manual-mount
+              name: cgroup-mountdir
+          {{- end }}
       containers:
       - name: egressd
         securityContext:
@@ -61,17 +80,13 @@ spec:
         - mountPath: /sys/fs/bpf
           mountPropagation: HostToContainer
           name: cilium-bpf-maps
-        - mountPath: /var/run/egressd
-          name: export-logs
         - mountPath: /hostproc/sys/net/netfilter
           name: netfilter
-          mountPropagation: Bidirectional
+          readOnly: true
         {{- if (get .Values.collector.extraArgs "ebpf-dns-tracer-enabled") }}
         - mountPath: /sys/fs/cgroup
           name: cgroup
-          mountPropagation: Bidirectional
-        - mountPath: /cgroup2-manual-mount
-          name: cgroup-mountdir
+          readOnly: true
         {{- end }}
         ports:
         - containerPort: {{.Values.collector.httpListenPort}}

charts/egressd/values-tilt.yaml

@@ -24,3 +24,6 @@ exporter:
       tilt_prom:
         prom_remote_write:
           url: "http://victoria-metrics-single-server:8428/api/v1/write"
+
+castai:
+  apiKey: "test"

charts/egressd/values.yaml

@@ -53,8 +53,15 @@ collector:
   # fsGroup: 2000
 
   containerSecurityContext:
-    privileged: true
+    privileged: false
     readOnlyRootFilesystem: true
+    capabilities:
+      drop:
+        - all
+      add:
+        - "NET_ADMIN"  # Allows managing network devices and configuring network interfaces.
+        - "SYS_PTRACE"  # Allows ptrace() of any process.
+        - "SYS_ADMIN"  # Allows a wide range of system administration tasks, such as mounting/unmounting filesystems, setting the domainname and hostname, and configuring process accounting and resource limits.
 
   resources:
     requests:

cmd/collector/main.go

@@ -43,6 +43,9 @@ var (
 	sendTrafficDelta       = flag.Bool("send-traffic-delta", false, "Send traffic delta between reads of conntrack entry. Traffic counter is sent by default")
 	ebpfDNSTracerEnabled   = flag.Bool("ebpf-dns-tracer-enabled", true, "Enable DNS tracer using eBPF")
 	ebpfDNSTracerQueueSize = flag.Int("ebpf-dns-tracer-queue-size", 1000, "Size of the queue for DNS tracer")
+	// Kubernetes requires container to run in privileged mode if Bidirectional mount is used.
+	// Actually it needs only SYS_ADMIN but see this issue See https://github.com/kubernetes/kubernetes/pull/117812
+	initMode = flag.Bool("init", false, "Run in init mode to setup conntrack accounting and mount cgroup v2")
 )
 
 // These should be set via `go build` during a release.
@@ -69,6 +72,13 @@ func main() {
 		return
 	}
 
+	if *initMode {
+		if err := runInit(log); err != nil {
+			log.Fatal(err)
+		}
+		return
+	}
+
 	if err := run(log); err != nil && !errors.Is(err, context.Canceled) {
 		log.Fatal(err)
 	}
@@ -194,6 +204,25 @@ func run(log logrus.FieldLogger) error {
 	return <-errc
 }
 
+// runInit runs once in init container.
+func runInit(log logrus.FieldLogger) error {
+	log.Infof("egressd: running init")
+	defer log.Infof("egressd: init done")
+
+	ciliumAvailable := conntrack.CiliumAvailable()
+	if !ciliumAvailable {
+		if err := conntrack.InitNetfilterAccounting(); err != nil {
+			return fmt.Errorf("init nf conntrack: %w", err)
+		}
+	}
+
+	if err := ebpf.InitCgroupv2(); err != nil {
+		return fmt.Errorf("init cgroupv2: %w", err)
+	}
+
+	return nil
+}
+
 func retrieveKubeConfig(log logrus.FieldLogger, kubepath string) (*rest.Config, error) {
 	if kubepath != "" {
 		data, err := os.ReadFile(kubepath)

conntrack/nf_conntrack.go

@@ -22,15 +22,11 @@ var (
 	accounting = "/hostproc/sys/net/netfilter/nf_conntrack_acct"
 )
 
-func initNetfilterAccounting() error {
+func InitNetfilterAccounting() error {
 	return os.WriteFile(accounting, []byte{'1'}, 0600)
 }
 
 func NewNetfilterClient(log logrus.FieldLogger) (Client, error) {
-	err := initNetfilterAccounting()
-	if err != nil {
-		return nil, fmt.Errorf("initing nf_conntrack_acct: %w", err)
-	}
 	nfct, err := ct.Open(&ct.Config{})
 	if err != nil {
 		return nil, fmt.Errorf("opening nfct: %w", err)

ebpf/tracer_linux.go

@@ -124,6 +124,16 @@ func IsKernelBTFAvailable() bool {
 	return err == nil
 }
 
+func InitCgroupv2() error {
+	_, err := detectCgroupPath()
+	if errors.Is(err, ErrCgroup2NotMounted) {
+		if err := mountCgroup2(); err != nil {
+			return fmt.Errorf("cgroup2 not mounted and failed to mount manually: %w", err)
+		}
+	}
+	return nil
+}
+
 func parseEvent(data []byte) (DNSEvent, error) {
 	packet := gopacket.NewPacket(
 		data,

ebpf/tracer_nolinux.go

@@ -17,3 +17,7 @@ func (t *Tracer) Events() <-chan DNSEvent {
 func IsKernelBTFAvailable() bool {
 	return false
 }
+
+func InitCgroupv2() error {
+	return nil
+}

go.mod

@@ -97,7 +97,7 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.mongodb.org/mongo-driver v1.10.2 // indirect
 	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
-	go4.org/unsafe/assume-no-moving-gc v0.0.0-20230209150437-ee73d164e760 // indirect
+	go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 // indirect
 	golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.1.0 // indirect

go.sum

@@ -502,6 +502,8 @@ go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:
 go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
 go4.org/unsafe/assume-no-moving-gc v0.0.0-20230209150437-ee73d164e760 h1:gH0IO5GDYAcawu+ThKrvAofVTgJjYaoOZ5rrC4pS2Xw=
 go4.org/unsafe/assume-no-moving-gc v0.0.0-20230209150437-ee73d164e760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
+go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 h1:lGdhQUN/cnWdSH3291CUuxSEqc+AsGTiDxPP3r2J0l4=
+go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=