Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

crictl pods - many pods NotReady after reboot, on crictl inspect i get "state": "SANDBOX_NOTREADY", "netNamespaceClosed": "true" #10865

Open
UriZafrir opened this issue Oct 21, 2024 · 0 comments
Open

crictl pods - many pods NotReady after reboot, on crictl inspect i get "state": "SANDBOX_NOTREADY", "netNamespaceClosed": "true" #10865

UriZafrir opened this issue Oct 21, 2024 · 0 comments
Labels
area/cri Container Runtime Interface (CRI) kind/bug

Comments

@UriZafrir
Copy link

Description

I've opened an issue in K3s but I think it's also related to containrd.
k3s-io/k3s#11139
please see it if necessary.

crictl pods:
Image
however all pods in kubectl are ok:
Image

no empty files exist
find /var/lib/cni/ -size 0
/var/lib/cni/networks/cbr0/lock

this is containerd logs:
https://gist.github.com/UriZafrir/6844efe214678a087e3b5c080f0f916c

crictl inspectp gives:
"state": "SANDBOX_NOTREADY":
"netNamespaceClosed": true,
pod has no ip.

this is true for all NotReady pods.
crictl inspectp:

{
  "status": {
    "id": "cf46d31391b99e9ca5f9ca29f4db027108e170f164c730079f8e72dc9abbacfa",
    "metadata": {
      "attempt": 0,
      "name": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
      "namespace": "kube-system",
      "uid": "85caaad1-0aa6-44e1-8a60-49fe06608ca3"
    },
    "state": "SANDBOX_NOTREADY",
    "createdAt": "2024-10-21T07:07:55.565102285Z",
    "network": {
      "additionalIps": [],
      "ip": ""
    },
    "linux": {
      "namespaces": {
        "options": {
          "ipc": "POD",
          "network": "POD",
          "pid": "CONTAINER",
          "targetId": "",
          "usernsOptions": null
        }
      }
    },
    "labels": {
      "app": "svclb-xxx-bucket-service-9a49bf46",
      "controller-revision-hash": "849b6c5d8",
      "io.kubernetes.pod.name": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
      "io.kubernetes.pod.namespace": "kube-system",
      "io.kubernetes.pod.uid": "85caaad1-0aa6-44e1-8a60-49fe06608ca3",
      "pod-template-generation": "1",
      "svccontroller.k3s.cattle.io/svcname": "xxx-bucket-service",
      "svccontroller.k3s.cattle.io/svcnamespace": "xxx"
    },
    "annotations": {
      "kubernetes.io/config.seen": "2024-10-21T07:07:55.290397677Z",
      "kubernetes.io/config.source": "api"
    },
    "runtimeHandler": ""
  },
  "info": {
    "pid": 0,
    "processStatus": "deleted",
    "netNamespaceClosed": true,
    "image": "docker.io/rancher/mirrored-pause:3.6",
    "snapshotKey": "cf46d31391b99e9ca5f9ca29f4db027108e170f164c730079f8e72dc9abbacfa",
    "snapshotter": "overlayfs",
    "runtimeHandler": "",
    "runtimeType": "io.containerd.runc.v2",
    "runtimeOptions": {
      "systemd_cgroup": true
    },
    "config": {
      "metadata": {
        "name": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
        "uid": "85caaad1-0aa6-44e1-8a60-49fe06608ca3",
        "namespace": "kube-system"
      },
      "hostname": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
      "log_directory": "/var/log/pods/kube-system_svclb-xxx-bucket-service-9a49bf46-jg2h6_85caaad1-0aa6-44e1-8a60-49fe06608ca3",
      "dns_config": {
        "servers": [
          "10.43.0.10"
        ],
        "searches": [
          "kube-system.svc.cluster.local",
          "svc.cluster.local",
          "cluster.local"
        ],
        "options": [
          "ndots:5"
        ]
      },
      "port_mappings": [
        {
          "container_port": 5000,
          "host_port": 5000
        }
      ],
      "labels": {
        "app": "svclb-xxx-bucket-service-9a49bf46",
        "controller-revision-hash": "849b6c5d8",
        "io.kubernetes.pod.name": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
        "io.kubernetes.pod.namespace": "kube-system",
        "io.kubernetes.pod.uid": "85caaad1-0aa6-44e1-8a60-49fe06608ca3",
        "pod-template-generation": "1",
        "svccontroller.k3s.cattle.io/svcname": "xxx-bucket-service",
        "svccontroller.k3s.cattle.io/svcnamespace": "xxx"
      },
      "annotations": {
        "kubernetes.io/config.seen": "2024-10-21T07:07:55.290397677Z",
        "kubernetes.io/config.source": "api"
      },
      "linux": {
        "cgroup_parent": "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod85caaad1_0aa6_44e1_8a60_49fe06608ca3.slice",
        "security_context": {
          "namespace_options": {
            "pid": 1
          },
          "seccomp": {}
        },
        "sysctls": {
          "net.ipv4.ip_forward": "1",
          "net.ipv4.ip_unprivileged_port_start": "0",
          "net.ipv4.ping_group_range": "0 2147483647"
        },
        "overhead": {},
        "resources": {
          "cpu_period": 100000,
          "cpu_shares": 2
        }
      }
    },
    "runtimeSpec": {
      "ociVersion": "1.2.0",
      "process": {
        "user": {
          "uid": 65535,
          "gid": 65535,
          "additionalGids": [
            65535
          ]
        },
        "args": [
          "/pause"
        ],
        "env": [
          "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
        ],
        "cwd": "/",
        "capabilities": {
          "bounding": [
            "CAP_CHOWN",
            "CAP_DAC_OVERRIDE",
            "CAP_FSETID",
            "CAP_FOWNER",
            "CAP_MKNOD",
            "CAP_NET_RAW",
            "CAP_SETGID",
            "CAP_SETUID",
            "CAP_SETFCAP",
            "CAP_SETPCAP",
            "CAP_NET_BIND_SERVICE",
            "CAP_SYS_CHROOT",
            "CAP_KILL",
            "CAP_AUDIT_WRITE"
          ],
          "effective": [
            "CAP_CHOWN",
            "CAP_DAC_OVERRIDE",
            "CAP_FSETID",
            "CAP_FOWNER",
            "CAP_MKNOD",
            "CAP_NET_RAW",
            "CAP_SETGID",
            "CAP_SETUID",
            "CAP_SETFCAP",
            "CAP_SETPCAP",
            "CAP_NET_BIND_SERVICE",
            "CAP_SYS_CHROOT",
            "CAP_KILL",
            "CAP_AUDIT_WRITE"
          ],
          "permitted": [
            "CAP_CHOWN",
            "CAP_DAC_OVERRIDE",
            "CAP_FSETID",
            "CAP_FOWNER",
            "CAP_MKNOD",
            "CAP_NET_RAW",
            "CAP_SETGID",
            "CAP_SETUID",
            "CAP_SETFCAP",
            "CAP_SETPCAP",
            "CAP_NET_BIND_SERVICE",
            "CAP_SYS_CHROOT",
            "CAP_KILL",
            "CAP_AUDIT_WRITE"
          ]
        },
        "noNewPrivileges": true,
        "oomScoreAdj": -998
      },
      "root": {
        "path": "rootfs",
        "readonly": true
      },
      "hostname": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
      "mounts": [
        {
          "destination": "/proc",
          "type": "proc",
          "source": "proc",
          "options": [
            "nosuid",
            "noexec",
            "nodev"
          ]
        },
        {
          "destination": "/dev",
          "type": "tmpfs",
          "source": "tmpfs",
          "options": [
            "nosuid",
            "strictatime",
            "mode=755",
            "size=65536k"
          ]
        },
        {
          "destination": "/dev/pts",
          "type": "devpts",
          "source": "devpts",
          "options": [
            "nosuid",
            "noexec",
            "newinstance",
            "ptmxmode=0666",
            "mode=0620",
            "gid=5"
          ]
        },
        {
          "destination": "/dev/mqueue",
          "type": "mqueue",
          "source": "mqueue",
          "options": [
            "nosuid",
            "noexec",
            "nodev"
          ]
        },
        {
          "destination": "/sys",
          "type": "sysfs",
          "source": "sysfs",
          "options": [
            "nosuid",
            "noexec",
            "nodev",
            "ro"
          ]
        },
        {
          "destination": "/dev/shm",
          "type": "bind",
          "source": "/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/cf46d31391b99e9ca5f9ca29f4db027108e170f164c730079f8e72dc9abbacfa/shm",
          "options": [
            "rbind",
            "ro",
            "nosuid",
            "nodev",
            "noexec"
          ]
        },
        {
          "destination": "/etc/resolv.conf",
          "type": "bind",
          "source": "/var/lib/rancher/k3s/agent/containerd/io.containerd.grpc.v1.cri/sandboxes/cf46d31391b99e9ca5f9ca29f4db027108e170f164c730079f8e72dc9abbacfa/resolv.conf",
          "options": [
            "rbind",
            "ro",
            "nosuid",
            "nodev",
            "noexec"
          ]
        }
      ],
      "annotations": {
        "io.kubernetes.cri.container-type": "sandbox",
        "io.kubernetes.cri.sandbox-cpu-period": "100000",
        "io.kubernetes.cri.sandbox-cpu-quota": "0",
        "io.kubernetes.cri.sandbox-cpu-shares": "2",
        "io.kubernetes.cri.sandbox-id": "cf46d31391b99e9ca5f9ca29f4db027108e170f164c730079f8e72dc9abbacfa",
        "io.kubernetes.cri.sandbox-log-directory": "/var/log/pods/kube-system_svclb-xxx-bucket-service-9a49bf46-jg2h6_85caaad1-0aa6-44e1-8a60-49fe06608ca3",
        "io.kubernetes.cri.sandbox-memory": "0",
        "io.kubernetes.cri.sandbox-name": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
        "io.kubernetes.cri.sandbox-namespace": "kube-system",
        "io.kubernetes.cri.sandbox-uid": "85caaad1-0aa6-44e1-8a60-49fe06608ca3"
      },
      "linux": {
        "sysctl": {
          "net.ipv4.ip_forward": "1",
          "net.ipv4.ip_unprivileged_port_start": "0",
          "net.ipv4.ping_group_range": "0 2147483647"
        },
        "resources": {
          "devices": [
            {
              "allow": false,
              "access": "rwm"
            }
          ],
          "cpu": {
            "shares": 2
          }
        },
        "cgroupsPath": "kubepods-besteffort-pod85caaad1_0aa6_44e1_8a60_49fe06608ca3.slice:cri-containerd:cf46d31391b99e9ca5f9ca29f4db027108e170f164c730079f8e72dc9abbacfa",
        "namespaces": [
          {
            "type": "pid"
          },
          {
            "type": "ipc"
          },
          {
            "type": "uts"
          },
          {
            "type": "mount"
          },
          {
            "type": "network",
            "path": "/var/run/netns/cni-769e973a-ba62-0965-fbfb-1daf0f978bb7"
          }
        ],
        "seccomp": {
          "defaultAction": "SCMP_ACT_ERRNO",
          "architectures": [
            "SCMP_ARCH_ARM",
            "SCMP_ARCH_AARCH64"
          ],
          "syscalls": [
            {
              "names": [
                "accept",
                "accept4",
                "access",
                "adjtimex",
                "alarm",
                "bind",
                "brk",
                "cachestat",
                "capget",
                "capset",
                "chdir",
                "chmod",
                "chown",
                "chown32",
                "clock_adjtime",
                "clock_adjtime64",
                "clock_getres",
                "clock_getres_time64",
                "clock_gettime",
                "clock_gettime64",
                "clock_nanosleep",
                "clock_nanosleep_time64",
                "close",
                "close_range",
                "connect",
                "copy_file_range",
                "creat",
                "dup",
                "dup2",
                "dup3",
                "epoll_create",
                "epoll_create1",
                "epoll_ctl",
                "epoll_ctl_old",
                "epoll_pwait",
                "epoll_pwait2",
                "epoll_wait",
                "epoll_wait_old",
                "eventfd",
                "eventfd2",
                "execve",
                "execveat",
                "exit",
                "exit_group",
                "faccessat",
                "faccessat2",
                "fadvise64",
                "fadvise64_64",
                "fallocate",
                "fanotify_mark",
                "fchdir",
                "fchmod",
                "fchmodat",
                "fchmodat2",
                "fchown",
                "fchown32",
                "fchownat",
                "fcntl",
                "fcntl64",
                "fdatasync",
                "fgetxattr",
                "flistxattr",
                "flock",
                "fork",
                "fremovexattr",
                "fsetxattr",
                "fstat",
                "fstat64",
                "fstatat64",
                "fstatfs",
                "fstatfs64",
                "fsync",
                "ftruncate",
                "ftruncate64",
                "futex",
                "futex_requeue",
                "futex_time64",
                "futex_wait",
                "futex_waitv",
                "futex_wake",
                "futimesat",
                "getcpu",
                "getcwd",
                "getdents",
                "getdents64",
                "getegid",
                "getegid32",
                "geteuid",
                "geteuid32",
                "getgid",
                "getgid32",
                "getgroups",
                "getgroups32",
                "getitimer",
                "getpeername",
                "getpgid",
                "getpgrp",
                "getpid",
                "getppid",
                "getpriority",
                "getrandom",
                "getresgid",
                "getresgid32",
                "getresuid",
                "getresuid32",
                "getrlimit",
                "get_robust_list",
                "getrusage",
                "getsid",
                "getsockname",
                "getsockopt",
                "get_thread_area",
                "gettid",
                "gettimeofday",
                "getuid",
                "getuid32",
                "getxattr",
                "inotify_add_watch",
                "inotify_init",
                "inotify_init1",
                "inotify_rm_watch",
                "io_cancel",
                "ioctl",
                "io_destroy",
                "io_getevents",
                "io_pgetevents",
                "io_pgetevents_time64",
                "ioprio_get",
                "ioprio_set",
                "io_setup",
                "io_submit",
                "io_uring_enter",
                "io_uring_register",
                "io_uring_setup",
                "ipc",
                "kill",
                "landlock_add_rule",
                "landlock_create_ruleset",
                "landlock_restrict_self",
                "lchown",
                "lchown32",
                "lgetxattr",
                "link",
                "linkat",
                "listen",
                "listxattr",
                "llistxattr",
                "_llseek",
                "lremovexattr",
                "lseek",
                "lsetxattr",
                "lstat",
                "lstat64",
                "madvise",
                "membarrier",
                "memfd_create",
                "memfd_secret",
                "mincore",
                "mkdir",
                "mkdirat",
                "mknod",
                "mknodat",
                "mlock",
                "mlock2",
                "mlockall",
                "map_shadow_stack",
                "mmap",
                "mmap2",
                "mprotect",
                "mq_getsetattr",
                "mq_notify",
                "mq_open",
                "mq_timedreceive",
                "mq_timedreceive_time64",
                "mq_timedsend",
                "mq_timedsend_time64",
                "mq_unlink",
                "mremap",
                "msgctl",
                "msgget",
                "msgrcv",
                "msgsnd",
                "msync",
                "munlock",
                "munlockall",
                "munmap",
                "name_to_handle_at",
                "nanosleep",
                "newfstatat",
                "_newselect",
                "open",
                "openat",
                "openat2",
                "pause",
                "pidfd_open",
                "pidfd_send_signal",
                "pipe",
                "pipe2",
                "pkey_alloc",
                "pkey_free",
                "pkey_mprotect",
                "poll",
                "ppoll",
                "ppoll_time64",
                "prctl",
                "pread64",
                "preadv",
                "preadv2",
                "prlimit64",
                "process_mrelease",
                "pselect6",
                "pselect6_time64",
                "pwrite64",
                "pwritev",
                "pwritev2",
                "read",
                "readahead",
                "readlink",
                "readlinkat",
                "readv",
                "recv",
                "recvfrom",
                "recvmmsg",
                "recvmmsg_time64",
                "recvmsg",
                "remap_file_pages",
                "removexattr",
                "rename",
                "renameat",
                "renameat2",
                "restart_syscall",
                "rmdir",
                "rseq",
                "rt_sigaction",
                "rt_sigpending",
                "rt_sigprocmask",
                "rt_sigqueueinfo",
                "rt_sigreturn",
                "rt_sigsuspend",
                "rt_sigtimedwait",
                "rt_sigtimedwait_time64",
                "rt_tgsigqueueinfo",
                "sched_getaffinity",
                "sched_getattr",
                "sched_getparam",
                "sched_get_priority_max",
                "sched_get_priority_min",
                "sched_getscheduler",
                "sched_rr_get_interval",
                "sched_rr_get_interval_time64",
                "sched_setaffinity",
                "sched_setattr",
                "sched_setparam",
                "sched_setscheduler",
                "sched_yield",
                "seccomp",
                "select",
                "semctl",
                "semget",
                "semop",
                "semtimedop",
                "semtimedop_time64",
                "send",
                "sendfile",
                "sendfile64",
                "sendmmsg",
                "sendmsg",
                "sendto",
                "setfsgid",
                "setfsgid32",
                "setfsuid",
                "setfsuid32",
                "setgid",
                "setgid32",
                "setgroups",
                "setgroups32",
                "setitimer",
                "setpgid",
                "setpriority",
                "setregid",
                "setregid32",
                "setresgid",
                "setresgid32",
                "setresuid",
                "setresuid32",
                "setreuid",
                "setreuid32",
                "setrlimit",
                "set_robust_list",
                "setsid",
                "setsockopt",
                "set_thread_area",
                "set_tid_address",
                "setuid",
                "setuid32",
                "setxattr",
                "shmat",
                "shmctl",
                "shmdt",
                "shmget",
                "shutdown",
                "sigaltstack",
                "signalfd",
                "signalfd4",
                "sigprocmask",
                "sigreturn",
                "socketcall",
                "socketpair",
                "splice",
                "stat",
                "stat64",
                "statfs",
                "statfs64",
                "statx",
                "symlink",
                "symlinkat",
                "sync",
                "sync_file_range",
                "syncfs",
                "sysinfo",
                "tee",
                "tgkill",
                "time",
                "timer_create",
                "timer_delete",
                "timer_getoverrun",
                "timer_gettime",
                "timer_gettime64",
                "timer_settime",
                "timer_settime64",
                "timerfd_create",
                "timerfd_gettime",
                "timerfd_gettime64",
                "timerfd_settime",
                "timerfd_settime64",
                "times",
                "tkill",
                "truncate",
                "truncate64",
                "ugetrlimit",
                "umask",
                "uname",
                "unlink",
                "unlinkat",
                "utime",
                "utimensat",
                "utimensat_time64",
                "utimes",
                "vfork",
                "vmsplice",
                "wait4",
                "waitid",
                "waitpid",
                "write",
                "writev"
              ],
              "action": "SCMP_ACT_ALLOW"
            },
            {
              "names": [
                "socket"
              ],
              "action": "SCMP_ACT_ALLOW",
              "args": [
                {
                  "index": 0,
                  "value": 40,
                  "op": "SCMP_CMP_NE"
                }
              ]
            },
            {
              "names": [
                "personality"
              ],
              "action": "SCMP_ACT_ALLOW",
              "args": [
                {
                  "index": 0,
                  "value": 0,
                  "op": "SCMP_CMP_EQ"
                }
              ]
            },
            {
              "names": [
                "personality"
              ],
              "action": "SCMP_ACT_ALLOW",
              "args": [
                {
                  "index": 0,
                  "value": 8,
                  "op": "SCMP_CMP_EQ"
                }
              ]
            },
            {
              "names": [
                "personality"
              ],
              "action": "SCMP_ACT_ALLOW",
              "args": [
                {
                  "index": 0,
                  "value": 131072,
                  "op": "SCMP_CMP_EQ"
                }
              ]
            },
            {
              "names": [
                "personality"
              ],
              "action": "SCMP_ACT_ALLOW",
              "args": [
                {
                  "index": 0,
                  "value": 131080,
                  "op": "SCMP_CMP_EQ"
                }
              ]
            },
            {
              "names": [
                "personality"
              ],
              "action": "SCMP_ACT_ALLOW",
              "args": [
                {
                  "index": 0,
                  "value": 4294967295,
                  "op": "SCMP_CMP_EQ"
                }
              ]
            },
            {
              "names": [
                "process_vm_readv",
                "process_vm_writev",
                "ptrace"
              ],
              "action": "SCMP_ACT_ALLOW"
            },
            {
              "names": [
                "arm_fadvise64_64",
                "arm_sync_file_range",
                "sync_file_range2",
                "breakpoint",
                "cacheflush",
                "set_tls"
              ],
              "action": "SCMP_ACT_ALLOW"
            },
            {
              "names": [
                "chroot"
              ],
              "action": "SCMP_ACT_ALLOW"
            },
            {
              "names": [
                "clone"
              ],
              "action": "SCMP_ACT_ALLOW",
              "args": [
                {
                  "index": 0,
                  "value": 2114060288,
                  "op": "SCMP_CMP_MASKED_EQ"
                }
              ]
            },
            {
              "names": [
                "clone3"
              ],
              "action": "SCMP_ACT_ERRNO",
              "errnoRet": 38
            }
          ]
        },
        "maskedPaths": [
          "/proc/acpi",
          "/proc/asound",
          "/proc/kcore",
          "/proc/keys",
          "/proc/latency_stats",
          "/proc/timer_list",
          "/proc/timer_stats",
          "/proc/sched_debug",
          "/sys/firmware",
          "/sys/devices/virtual/powercap",
          "/proc/scsi"
        ],
        "readonlyPaths": [
          "/proc/bus",
          "/proc/fs",
          "/proc/irq",
          "/proc/sys",
          "/proc/sysrq-trigger"
        ]
      }
    },
    "cniResult": {
      "Interfaces": {
        "cni0": {
          "IPConfigs": null,
          "Mac": "42:42:e0:e7:48:d7",
          "Sandbox": ""
        },
        "eth0": {
          "IPConfigs": [
            {
              "IP": "10.42.0.10",
              "Gateway": "10.42.0.1"
            }
          ],
          "Mac": "8a:85:47:6a:8d:54",
          "Sandbox": "/var/run/netns/cni-769e973a-ba62-0965-fbfb-1daf0f978bb7"
        },
        "lo": {
          "IPConfigs": [
            {
              "IP": "127.0.0.1",
              "Gateway": ""
            },
            {
              "IP": "::1",
              "Gateway": ""
            }
          ],
          "Mac": "00:00:00:00:00:00",
          "Sandbox": "/var/run/netns/cni-769e973a-ba62-0965-fbfb-1daf0f978bb7"
        },
        "veth0270ece6": {
          "IPConfigs": null,
          "Mac": "5a:17:30:e1:db:0b",
          "Sandbox": ""
        }
      },
      "DNS": [
        {},
        {}
      ],
      "Routes": [
        {
          "dst": "10.42.0.0/16"
        },
        {
          "dst": "0.0.0.0/0",
          "gw": "10.42.0.1"
        }
      ]
    }
  }
}

Steps to reproduce the issue

  1. install k3s
  2. install metallb, two pods with helm:
helm install metallb metallb-chart/ --namespace metallb-system -f metallb-chart/values.yaml --debug --wait --timeout 10m
helm install xxxx helm-chart/ --namespace xxxx-f helm-chart/values-dev.yaml --debug --wait --timeout 60m
  1. reboot

Describe the results you received and expected

expected:
crictl pods showing all pods ready
received:
crictl pods showing multiple pods not ready

What version of containerd are you using?

ctr github.com/k3s-io/containerd v1.7.15-k3s1

Any other relevant information

uname -a

Linux tegra-ubuntu 5.10.120-tegra #11 SMP PREEMPT Wed Nov 1 13:11:49 CST 2023 aarch64 aarch64 aarch64 GNU/Linux

crictl info

crictl info
{
  "status": {
    "conditions": [
      {
        "type": "RuntimeReady",
        "status": true,
        "reason": "",
        "message": ""
      },
      {
        "type": "NetworkReady",
        "status": true,
        "reason": "",
        "message": ""
      },
      {
        "type": "ContainerdHasNoDeprecationWarnings",
        "status": false,
        "reason": "ContainerdHasDeprecationWarnings",
        "message": "{\"io.containerd.deprecation/cri-registry-configs\":\"The `configs` property of `[plugins.\\\"io.containerd.grpc.v1.cri\\\".registry]` is deprecated since containerd v1.5 and will be removed in containerd v2.0. Use `config_path` instead.\"}"
      }
    ]
  },
  "cniconfig": {
    "PluginDirs": [
      "/var/lib/rancher/k3s/data/d85c4d741f84e813ef6cbfbf45702297c423d3f1a286057b3b1096c16c63bcc2/bin"
    ],
    "PluginConfDir": "/var/lib/rancher/k3s/agent/etc/cni/net.d",
    "PluginMaxConfNum": 1,
    "Prefix": "eth",
    "Networks": [
      {
        "Config": {
          "Name": "cni-loopback",
          "CNIVersion": "0.3.1",
          "Plugins": [
            {
              "Network": {
                "type": "loopback",
                "ipam": {},
                "dns": {}
              },
              "Source": "{\"type\":\"loopback\"}"
            }
          ],
          "Source": "{\n\"cniVersion\": \"0.3.1\",\n\"name\": \"cni-loopback\",\n\"plugins\": [{\n  \"type\": \"loopback\"\n}]\n}"
        },
        "IFName": "lo"
      },
      {
        "Config": {
          "Name": "cbr0",
          "CNIVersion": "1.0.0",
          "Plugins": [
            {
              "Network": {
                "type": "flannel",
                "ipam": {},
                "dns": {}
              },
              "Source": "{\"delegate\":{\"forceAddress\":true,\"hairpinMode\":true,\"isDefaultGateway\":true},\"type\":\"flannel\"}"
            },
            {
              "Network": {
                "type": "portmap",
                "capabilities": {
                  "portMappings": true
                },
                "ipam": {},
                "dns": {}
              },
              "Source": "{\"capabilities\":{\"portMappings\":true},\"type\":\"portmap\"}"
            },
            {
              "Network": {
                "type": "bandwidth",
                "capabilities": {
                  "bandwidth": true
                },
                "ipam": {},
                "dns": {}
              },
              "Source": "{\"capabilities\":{\"bandwidth\":true},\"type\":\"bandwidth\"}"
            }
          ],
          "Source": "{\n  \"name\":\"cbr0\",\n  \"cniVersion\":\"1.0.0\",\n  \"plugins\":[\n    {\n      \"type\":\"flannel\",\n      \"delegate\":{\n        \"hairpinMode\":true,\n        \"forceAddress\":true,\n        \"isDefaultGateway\":true\n      }\n    },\n    {\n      \"type\":\"portmap\",\n      \"capabilities\":{\n        \"portMappings\":true\n      }\n    },\n    {\n      \"type\":\"bandwidth\",\n      \"capabilities\":{\n        \"bandwidth\":true\n      }\n    }\n  ]\n}\n"
        },
        "IFName": "eth0"
      }
    ]
  },
  "config": {
    "containerd": {
      "snapshotter": "overlayfs",
      "defaultRuntimeName": "runc",
      "defaultRuntime": {
        "runtimeType": "",
        "runtimePath": "",
        "runtimeEngine": "",
        "PodAnnotations": null,
        "ContainerAnnotations": null,
        "runtimeRoot": "",
        "options": null,
        "privileged_without_host_devices": false,
        "privileged_without_host_devices_all_devices_allowed": false,
        "baseRuntimeSpec": "",
        "cniConfDir": "",
        "cniMaxConfNum": 0,
        "snapshotter": "",
        "sandboxMode": ""
      },
      "untrustedWorkloadRuntime": {
        "runtimeType": "",
        "runtimePath": "",
        "runtimeEngine": "",
        "PodAnnotations": null,
        "ContainerAnnotations": null,
        "runtimeRoot": "",
        "options": null,
        "privileged_without_host_devices": false,
        "privileged_without_host_devices_all_devices_allowed": false,
        "baseRuntimeSpec": "",
        "cniConfDir": "",
        "cniMaxConfNum": 0,
        "snapshotter": "",
        "sandboxMode": ""
      },
      "runtimes": {
        "runc": {
          "runtimeType": "io.containerd.runc.v2",
          "runtimePath": "",
          "runtimeEngine": "",
          "PodAnnotations": null,
          "ContainerAnnotations": null,
          "runtimeRoot": "",
          "options": {
            "SystemdCgroup": true
          },
          "privileged_without_host_devices": false,
          "privileged_without_host_devices_all_devices_allowed": false,
          "baseRuntimeSpec": "",
          "cniConfDir": "",
          "cniMaxConfNum": 0,
          "snapshotter": "",
          "sandboxMode": "podsandbox"
        }
      },
      "noPivot": false,
      "disableSnapshotAnnotations": true,
      "discardUnpackedLayers": false,
      "ignoreBlockIONotEnabledErrors": false,
      "ignoreRdtNotEnabledErrors": false
    },
    "cni": {
      "binDir": "/var/lib/rancher/k3s/data/d85c4d741f84e813ef6cbfbf45702297c423d3f1a286057b3b1096c16c63bcc2/bin",
      "confDir": "/var/lib/rancher/k3s/agent/etc/cni/net.d",
      "maxConfNum": 1,
      "setupSerially": false,
      "confTemplate": "",
      "ipPref": ""
    },
    "registry": {
      "configPath": "/var/lib/rancher/k3s/agent/etc/containerd/certs.d",
      "mirrors": null,
      "configs": {
        "xxxx.azurecr.io": {
          "auth": {
            "username": "xxxx",
            "password": "xxxx",
            "auth": "",
            "identitytoken": ""
          },
          "tls": null
        }
      },
      "auths": null,
      "headers": null
    },
    "imageDecryption": {
      "keyModel": "node"
    },
    "disableTCPService": true,
    "streamServerAddress": "127.0.0.1",
    "streamServerPort": "10010",
    "streamIdleTimeout": "4h0m0s",
    "enableSelinux": false,
    "selinuxCategoryRange": 1024,
    "sandboxImage": "rancher/mirrored-pause:3.6",
    "statsCollectPeriod": 10,
    "systemdCgroup": false,
    "enableTLSStreaming": false,
    "x509KeyPairStreaming": {
      "tlsCertFile": "",
      "tlsKeyFile": ""
    },
    "maxContainerLogSize": 16384,
    "disableCgroup": false,
    "disableApparmor": false,
    "restrictOOMScoreAdj": false,
    "maxConcurrentDownloads": 3,
    "disableProcMount": false,
    "unsetSeccompProfile": "",
    "tolerateMissingHugetlbController": true,
    "disableHugetlbController": true,
    "device_ownership_from_security_context": false,
    "ignoreImageDefinedVolumes": false,
    "netnsMountsUnderStateDir": false,
    "enableUnprivilegedPorts": true,
    "enableUnprivilegedICMP": true,
    "enableCDI": false,
    "cdiSpecDirs": [
      "/etc/cdi",
      "/var/run/cdi"
    ],
    "imagePullProgressTimeout": "5m0s",
    "drainExecSyncIOTimeout": "0s",
    "imagePullWithSyncFs": false,
    "ignoreDeprecationWarnings": null,
    "containerdRootDir": "/var/lib/rancher/k3s/agent/containerd",
    "containerdEndpoint": "/run/k3s/containerd/containerd.sock",
    "rootDir": "/var/lib/rancher/k3s/agent/containerd/io.containerd.grpc.v1.cri",
    "stateDir": "/run/k3s/containerd/io.containerd.grpc.v1.cri"
  },
  "golang": "go1.22.2",
  "lastCNILoadStatus": "OK",
  "lastCNILoadStatus.default": "OK"
}

Show configuration if it is related to CRI plugin.

cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml

# File generated by k3s. DO NOT EDIT. Use config.toml.tmpl instead.
version = 2

[plugins."io.containerd.internal.v1.opt"]
  path = "/var/lib/rancher/k3s/agent/containerd"
[plugins."io.containerd.grpc.v1.cri"]
  stream_server_address = "127.0.0.1"
  stream_server_port = "10010"
  enable_selinux = false
  enable_unprivileged_ports = true
  enable_unprivileged_icmp = true
  sandbox_image = "rancher/mirrored-pause:3.6"

[plugins."io.containerd.grpc.v1.cri".containerd]
  snapshotter = "overlayfs"
  disable_snapshot_annotations = true



[plugins."io.containerd.grpc.v1.cri".cni]
  bin_dir = "/var/lib/rancher/k3s/data/d85c4d741f84e813ef6cbfbf45702297c423d3f1a286057b3b1096c16c63bcc2/bin"
  conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d"


[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
  runtime_type = "io.containerd.runc.v2"

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
  SystemdCgroup = true

[plugins."io.containerd.grpc.v1.cri".registry]
  config_path = "/var/lib/rancher/k3s/agent/etc/containerd/certs.d"




[plugins."io.containerd.grpc.v1.cri".registry.configs."xxxx.azurecr.io".auth]
  username = "xxxx"
  password = "xxxx"
@dosubot dosubot bot added the area/cri Container Runtime Interface (CRI) label Oct 21, 2024
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
area/cri Container Runtime Interface (CRI) kind/bug
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
1 participant
@UriZafrir