From 337292a8d2272c6315dfc8a0c2e5cc1f12961241 Mon Sep 17 00:00:00 2001 From: Edgar Lee Date: Wed, 14 Feb 2024 05:27:28 -0500 Subject: [PATCH] Add nix-snapshotter support to the embedded containerd Signed-off-by: Edgar Lee --- go.mod | 4 +++- go.sum | 10 ++++++++-- pkg/agent/config/config.go | 6 ++++++ pkg/agent/containerd/config_linux.go | 5 +++++ pkg/agent/containerd/config_windows.go | 4 ++++ pkg/agent/templates/templates_linux.go | 10 +++++++++- pkg/containerd/builtins_linux.go | 1 + pkg/containerd/utility_linux.go | 5 +++++ pkg/containerd/utility_windows.go | 4 ++++ 9 files changed, 45 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 001d9bcfc896..8622cf07254c 100644 --- a/go.mod +++ b/go.mod @@ -135,6 +135,7 @@ require ( github.com/opencontainers/runc v1.1.12 github.com/opencontainers/selinux v1.11.0 github.com/otiai10/copy v1.7.0 + github.com/pdtpartners/nix-snapshotter v0.1.2-0.20240214230725-1c25427973e0 github.com/pkg/errors v0.9.1 github.com/prometheus/common v0.45.0 github.com/rancher/dynamiclistener v0.3.6 @@ -184,6 +185,7 @@ require ( require ( cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect + dario.cat/mergo v1.0.0 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect @@ -219,7 +221,7 @@ require ( github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cilium/ebpf v0.9.1 // indirect - github.com/container-orchestrated-devices/container-device-interface v0.5.4 // indirect + github.com/container-orchestrated-devices/container-device-interface v0.6.0 // indirect github.com/container-storage-interface/spec v1.8.0 // indirect github.com/containerd/btrfs/v2 v2.0.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect diff --git a/go.sum b/go.sum index 82dbd71ff946..8f2863ffaa5e 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,8 @@ cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27 cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= @@ -380,8 +382,9 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+g github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/container-orchestrated-devices/container-device-interface v0.5.4 h1:PqQGqJqQttMP5oJ/qNGEg8JttlHqGY3xDbbcKb5T9E8= github.com/container-orchestrated-devices/container-device-interface v0.5.4/go.mod h1:DjE95rfPiiSmG7uVXtg0z6MnPm/Lx4wxKCIts0ZE0vg= +github.com/container-orchestrated-devices/container-device-interface v0.6.0 h1:aWwcz/Ep0Fd7ZuBjQGjU/jdPloM7ydhMW13h85jZNvk= +github.com/container-orchestrated-devices/container-device-interface v0.6.0/go.mod h1:OQlgtJtDrOxSQ1BWODC8OZK1tzi9W69wek+Jy17ndzo= github.com/container-storage-interface/spec v1.8.0 h1:D0vhF3PLIZwlwZEf2eNbpujGCNwspwTYf2idJRJx4xI= github.com/container-storage-interface/spec v1.8.0/go.mod h1:ROLik+GhPslwwWRNFF1KasPzroNARibH2rfz1rkg4H0= github.com/containerd/aufs v1.0.0 h1:2oeJiwX5HstO7shSrPZjrohJZLzK36wvpdmzDRkL/LY= @@ -1388,6 +1391,8 @@ github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pdtpartners/nix-snapshotter v0.1.2-0.20240214230725-1c25427973e0 h1:d3mKjpcgmkgaGl6RiMRabjSqo8bHdavPonv6P/zaqpw= +github.com/pdtpartners/nix-snapshotter v0.1.2-0.20240214230725-1c25427973e0/go.mod h1:MKa+V5fH15XmLCDt+s8qRQeIAaadaJ3/4+/oD7f0K0k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -2205,8 +2210,9 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 4a16edd68a32..ac744846dc4e 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -556,6 +556,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.Containerd.Root) } nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" + case "nix": + if err := containerd.NixSupported(nodeConfig.Containerd.Root); err != nil { + return nil, errors.Wrapf(err, "\"nix\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"", + nodeConfig.Containerd.Root) + } + nodeConfig.AgentConfig.ImageServiceSocket = "/run/k3s/nix-snapshotter/nix-snapshotter.sock" } } else { nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ContainerRuntimeEndpoint diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go index 9808949dd3ac..9a0f9fe5d449 100644 --- a/pkg/agent/containerd/config_linux.go +++ b/pkg/agent/containerd/config_linux.go @@ -16,6 +16,7 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/version" "github.com/opencontainers/runc/libcontainer/userns" + "github.com/pdtpartners/nix-snapshotter/pkg/nix" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" @@ -117,3 +118,7 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return stargz.Supported(root) } + +func NixSupported(root string) error { + return nix.Supported(root) +} diff --git a/pkg/agent/containerd/config_windows.go b/pkg/agent/containerd/config_windows.go index 4172ad315e8f..dfbd3919ba11 100644 --- a/pkg/agent/containerd/config_windows.go +++ b/pkg/agent/containerd/config_windows.go @@ -64,3 +64,7 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return errors.Wrapf(util3.ErrUnsupportedPlatform, "stargz is not supported") } + +func NixSupported(root string) error { + return errors.Wrapf(util3.ErrUnsupportedPlatform, "nix is not supported") +} diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index 0df107abaae7..16257eb3c5e8 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -35,7 +35,7 @@ version = 2 {{- if .NodeConfig.AgentConfig.Snapshotter }} [plugins."io.containerd.grpc.v1.cri".containerd] snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}" - disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}} + disable_snapshot_annotations = {{ if or (eq .NodeConfig.AgentConfig.Snapshotter "stargz") (eq .NodeConfig.AgentConfig.Snapshotter "nix") }}false{{else}}true{{end}} {{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}} {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} {{ if .NodeConfig.AgentConfig.ImageServiceSocket }} @@ -75,6 +75,14 @@ enable_keychain = true {{end}} {{end}} {{end}} +{{ if eq .NodeConfig.AgentConfig.Snapshotter "nix" }} +[plugins."io.containerd.snapshotter.v1.nix"] +address = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}" +image_service.enable = true +[[plugins."io.containerd.transfer.v1.local".unpack_config]] +platform = "linux/amd64" +snapshotter = "nix" +{{end}} {{end}} {{- if not .NodeConfig.NoFlannel }} diff --git a/pkg/containerd/builtins_linux.go b/pkg/containerd/builtins_linux.go index a0ea4dc49613..98c443625da5 100644 --- a/pkg/containerd/builtins_linux.go +++ b/pkg/containerd/builtins_linux.go @@ -32,4 +32,5 @@ import ( _ "github.com/containerd/fuse-overlayfs-snapshotter/plugin" _ "github.com/containerd/stargz-snapshotter/service/plugin" _ "github.com/containerd/zfs/plugin" + _ "github.com/pdtpartners/nix-snapshotter/pkg/plugin" ) diff --git a/pkg/containerd/utility_linux.go b/pkg/containerd/utility_linux.go index 76ff569b41c0..b6f2d0cfebb4 100644 --- a/pkg/containerd/utility_linux.go +++ b/pkg/containerd/utility_linux.go @@ -6,6 +6,7 @@ import ( "github.com/containerd/containerd/snapshots/overlay/overlayutils" fuseoverlayfs "github.com/containerd/fuse-overlayfs-snapshotter" stargz "github.com/containerd/stargz-snapshotter/service" + "github.com/pdtpartners/nix-snapshotter/pkg/nix" ) func OverlaySupported(root string) error { @@ -19,3 +20,7 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return stargz.Supported(root) } + +func NixSupported(root string) error { + return nix.Supported(root) +} diff --git a/pkg/containerd/utility_windows.go b/pkg/containerd/utility_windows.go index 13a58e55bfdb..6f6833177fa2 100644 --- a/pkg/containerd/utility_windows.go +++ b/pkg/containerd/utility_windows.go @@ -19,3 +19,7 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return errors.Wrapf(util2.ErrUnsupportedPlatform, "stargz is not supported") } + +func NixSupported(root string) error { + return errors.Wrapf(util2.ErrUnsupportedPlatform, "nix is not supported") +}