From ab340f4d64e95aee51d8805ed1666134a558b0dd Mon Sep 17 00:00:00 2001 From: Edgar Lee Date: Wed, 14 Feb 2024 05:27:28 -0500 Subject: [PATCH] Add nix-snapshotter support to the embedded containerd Signed-off-by: Edgar Lee --- go.mod | 1 + go.sum | 5 ++++- pkg/agent/config/config.go | 6 ++++++ pkg/agent/containerd/config_linux.go | 5 +++++ pkg/agent/containerd/config_windows.go | 4 ++++ pkg/agent/templates/templates_linux.go | 10 +++++++++- pkg/containerd/builtins_linux.go | 1 + pkg/containerd/utility_linux.go | 5 +++++ pkg/containerd/utility_windows.go | 4 ++++ 9 files changed, 39 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index f32eedf0581c..73979f08ed09 100644 --- a/go.mod +++ b/go.mod @@ -115,6 +115,7 @@ require ( github.com/opencontainers/runc v1.1.13 github.com/opencontainers/selinux v1.11.0 github.com/otiai10/copy v1.7.0 + github.com/pdtpartners/nix-snapshotter v0.2.1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.20.1 github.com/prometheus/common v0.55.0 diff --git a/go.sum b/go.sum index bd68b1b1548a..ef8d051a6ed5 100644 --- a/go.sum +++ b/go.sum @@ -1388,6 +1388,8 @@ github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pdtpartners/nix-snapshotter v0.2.1 h1:NVmZbcDvD5UWNs4oF4vMmhRTlcf596+/fSPb6koiGKs= +github.com/pdtpartners/nix-snapshotter v0.2.1/go.mod h1:MKa+V5fH15XmLCDt+s8qRQeIAaadaJ3/4+/oD7f0K0k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -2229,8 +2231,9 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 795618b03b83..9eed5d4c82de 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -594,6 +594,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.Containerd.Root) } nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" + case "nix": + if err := containerd.NixSupported(nodeConfig.Containerd.Root); err != nil { + return nil, errors.Wrapf(err, "\"nix\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"", + nodeConfig.Containerd.Root) + } + nodeConfig.AgentConfig.ImageServiceSocket = "/run/k3s/nix-snapshotter/nix-snapshotter.sock" } } else { nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ContainerRuntimeEndpoint diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go index 5bd7df655a62..c3f3afefcf52 100644 --- a/pkg/agent/containerd/config_linux.go +++ b/pkg/agent/containerd/config_linux.go @@ -16,6 +16,7 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/version" "github.com/opencontainers/runc/libcontainer/userns" + "github.com/pdtpartners/nix-snapshotter/pkg/nix" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" @@ -117,3 +118,7 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return stargz.Supported(root) } + +func NixSupported(root string) error { + return nix.Supported(root) +} diff --git a/pkg/agent/containerd/config_windows.go b/pkg/agent/containerd/config_windows.go index 5e102401e5eb..6074f5f07536 100644 --- a/pkg/agent/containerd/config_windows.go +++ b/pkg/agent/containerd/config_windows.go @@ -64,3 +64,7 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return errors.Wrapf(util3.ErrUnsupportedPlatform, "stargz is not supported") } + +func NixSupported(root string) error { + return errors.Wrapf(util3.ErrUnsupportedPlatform, "nix is not supported") +} diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index c064f6fcb4cd..1ded3238947d 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -35,7 +35,7 @@ version = 2 {{- if .NodeConfig.AgentConfig.Snapshotter }} [plugins."io.containerd.grpc.v1.cri".containerd] snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}" - disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}} + disable_snapshot_annotations = {{ if or (eq .NodeConfig.AgentConfig.Snapshotter "stargz") (eq .NodeConfig.AgentConfig.Snapshotter "nix") }}false{{else}}true{{end}} {{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}} {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} {{ if .NodeConfig.AgentConfig.ImageServiceSocket }} @@ -60,6 +60,14 @@ enable_keychain = true {{end}} {{end}} {{end}} +{{ if eq .NodeConfig.AgentConfig.Snapshotter "nix" }} +[plugins."io.containerd.snapshotter.v1.nix"] +address = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}" +image_service.enable = true +[[plugins."io.containerd.transfer.v1.local".unpack_config]] +platform = "linux/amd64" +snapshotter = "nix" +{{end}} {{end}} {{- if not .NodeConfig.NoFlannel }} diff --git a/pkg/containerd/builtins_linux.go b/pkg/containerd/builtins_linux.go index a0ea4dc49613..98c443625da5 100644 --- a/pkg/containerd/builtins_linux.go +++ b/pkg/containerd/builtins_linux.go @@ -32,4 +32,5 @@ import ( _ "github.com/containerd/fuse-overlayfs-snapshotter/plugin" _ "github.com/containerd/stargz-snapshotter/service/plugin" _ "github.com/containerd/zfs/plugin" + _ "github.com/pdtpartners/nix-snapshotter/pkg/plugin" ) diff --git a/pkg/containerd/utility_linux.go b/pkg/containerd/utility_linux.go index 76ff569b41c0..b6f2d0cfebb4 100644 --- a/pkg/containerd/utility_linux.go +++ b/pkg/containerd/utility_linux.go @@ -6,6 +6,7 @@ import ( "github.com/containerd/containerd/snapshots/overlay/overlayutils" fuseoverlayfs "github.com/containerd/fuse-overlayfs-snapshotter" stargz "github.com/containerd/stargz-snapshotter/service" + "github.com/pdtpartners/nix-snapshotter/pkg/nix" ) func OverlaySupported(root string) error { @@ -19,3 +20,7 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return stargz.Supported(root) } + +func NixSupported(root string) error { + return nix.Supported(root) +} diff --git a/pkg/containerd/utility_windows.go b/pkg/containerd/utility_windows.go index 13a58e55bfdb..6f6833177fa2 100644 --- a/pkg/containerd/utility_windows.go +++ b/pkg/containerd/utility_windows.go @@ -19,3 +19,7 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return errors.Wrapf(util2.ErrUnsupportedPlatform, "stargz is not supported") } + +func NixSupported(root string) error { + return errors.Wrapf(util2.ErrUnsupportedPlatform, "nix is not supported") +}