calico fails when creating pod

[jacy1101]

./kubetcd create pod nginxpersistent -t nginx -p randomentry  
Path Template:/registry/pods/default/nginx
Deserializing...
Tampering data...
Serializing...
Path injected: /registry/pods/default/randomentry
OK

root@k8smaster:~# kubectl describe pod randomentry
Name:             nginxpersistent
Namespace:        default
Priority:         0
Service Account:  default
Node:             192.168.8.155/192.168.8.155
Start Time:       Sun, 03 Dec 2023 17:17:36 +0800
Labels:           run=nginxpersistent
Annotations:      <none>
Status:           Running
IP:               172.20.16.135
IPs:
  IP:  172.20.16.135
Containers:
  nginxpersistent:
    Container ID:   containerd://3c4d2d3c87dc52e5add26ab40f2affc6a2914f79e7c00a601ef5fb2f3f075db6
    Image:          nginx
    Image ID:       docker.io/library/nginx@sha256:0d17b565c37bcbd895e9d92315a05c1c3c9a29f762b011a10c54a66cd53c9b31
    Port:           <none>
    Host Port:      <none>
    State:          Running
      Started:      Sun, 03 Dec 2023 17:24:31 +0800
    Ready:          True
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-b55gd (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  kube-api-access-b55gd:
    Type:               Projected (a volume that contains injected data from multiple sources)
    ConfigMapName:      kube-root-ca.crt
    ConfigMapOptional:  <nil>
    DownwardAPI:        true
QoS Class:              BestEffort
Node-Selectors:         <none>
Tolerations:            node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                        node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason                  Age               From     Message
  ----     ------                  ----              ----     -------
  Warning  FailedCreatePodSandBox  2m18s             kubelet  Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "f2e6be10e5ec2cdd5a3ebdb5932e93ff5ca79b9c8dd2d617e9d975e3b3162a35": plugin type="calico" failed (add): pods "nginxpersistent" not found
  Warning  FailedCreatePodSandBox  2m3s              kubelet  Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "ea8b4aa790fe5c594af183801382f7ad2b0304b22128ca940e018b8950350425": plugin type="calico" failed (add): pods "nginxpersistent" not found
  Warning  FailedCreatePodSandBox  108s              kubelet  Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "ee145b8a4c537c70a49c3c564e70bbee114564c8535f41e709746adae4b9f707": plugin type="calico" failed (add): pods "nginxpersistent" not found
  Warning  FailedCreatePodSandBox  96s               kubelet  Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "4d53861d8f173138a4cf22965627849270ae0ec551bec081b66cce344d20cb49": plugin type="calico" failed (add): pods "nginxpersistent" not found
  Warning  FailedCreatePodSandBox  81s               kubelet  Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "80d7a9d0aabfada5239234813edc83e182000a1eab504e182544b1b3caeae8bd": plugin type="calico" failed (add): pods "nginxpersistent" not found
  Warning  FailedCreatePodSandBox  68s               kubelet  Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "f85d812b0af455afff7f4e6feabe2bf799b7b2b0c622cf9298dfabe95b526686": plugin type="calico" failed (add): pods "nginxpersistent" not found
  Warning  FailedCreatePodSandBox  54s               kubelet  Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "ae650d34a502c1d624c3aceafeba74c24b2d6154c80101fe7bd4ab606bc4cf8a": plugin type="calico" failed (add): pods "nginxpersistent" not found
  Warning  FailedCreatePodSandBox  42s               kubelet  Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "1b68f114af65f62b7e107ee4ad107821a6703f94a4d9d34a54b86105bbbc4856": plugin type="calico" failed (add): pods "nginxpersistent" not found
  Warning  FailedCreatePodSandBox  28s               kubelet  Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "a9fb7b57426170680d1eb4fddbe3e7c1ba25096b0cb6ff0ef02373b73254f154": plugin type="calico" failed (add): pods "nginxpersistent" not found
  Warning  FailedCreatePodSandBox  1s (x2 over 15s)  kubelet  (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "c78d994bf823779e0aa55953d353e496a0a14666d66f372491eab2e716476f13": plugin type="calico" failed (add): pods "nginxpersistent" not found

No new pods are started

root@k8smaster:~# crictl ps 
CONTAINER           IMAGE               CREATED             STATE               NAME                        ATTEMPT             POD ID              POD
3c4d2d3c87dc5       605c77e624ddb       7 minutes ago       Running             nginx                       0                   1cc65e1183305       nginx
6576fe716e123       115053965e86b       32 minutes ago      Running             dashboard-metrics-scraper   0                   f3595988cff8f       dashboard-metrics-scraper-57566685b4-f78dd
257fb8f288daa       07655ddf2eebe       32 minutes ago      Running             kubernetes-dashboard        0                   350b1fced40f1       kubernetes-dashboard-57db9bfd5b-2sbwb
a872c956237dd       f965999d664b1       32 minutes ago      Running             metrics-server              0                   8d52485c6cdd9       metrics-server-6bd9f986fc-qc8rk
c3125283ec0eb       7b3b529c5a5ac       32 minutes ago      Running             node-cache                  0                   7dcb09366b649       node-local-dns-dxwp6
7caddf12a0429       5185b96f0becf       32 minutes ago      Running             coredns                     0                   ce24bc392eab3       coredns-6665999d97-rn8ht
909f305b0002d       b6e6ee0788f20       33 minutes ago      Running             calico-node                 0                   0051e9bf6c15d       calico-node-cxwmk
b9ee759a90070       ea5536b1fa4a8       33 minutes ago      Running             calico-kube-controllers     0                   50886bda9b6b1       calico-kube-controllers-89b744d6c-9fpsd

[jacy1101]

root@k8smaster:~# kubectl version
WARNING: This version information is deprecated and will be replaced with the output from kubectl version --short.  Use --output=yaml|json to get the full version.
Client Version: version.Info{Major:"1", Minor:"26", GitVersion:"v1.26.0", GitCommit:"b46a3f887ca979b1a5d14fd39cb1af43e7e5d12d", GitTreeState:"clean", BuildDate:"2022-12-08T19:58:30Z", GoVersion:"go1.19.4", Compiler:"gc", Platform:"linux/amd64"}
Kustomize Version: v4.5.7
Server Version: version.Info{Major:"1", Minor:"26", GitVersion:"v1.26.0", GitCommit:"b46a3f887ca979b1a5d14fd39cb1af43e7e5d12d", GitTreeState:"clean", BuildDate:"2022-12-08T19:51:45Z", GoVersion:"go1.19.4", Compiler:"gc", Platform:"linux/amd64"}

root@k8smaster:~# calicoctl version
Client Version:    v3.23.5
Git commit:        9e0398360
Cluster Version:   v3.23.5
Cluster Type:      k8s,bgp

root@k8smaster:~# etcdctl version
etcdctl version: 3.5.5
API version: 3.5

[jacy1101]

It works if I use flannel

root@k8smaster:~# kubectl describe pod nginxpersistent
Pod 'nginxpersistent': error 'pods "nginxpersistent" not found', but found events.
Events:
  Type    Reason   Age   From     Message
  ----    ------   ----  ----     -------
  Normal  Pulling  12m   kubelet  Pulling image "nginx"
  Normal  Pulled   11m   kubelet  Successfully pulled image "nginx" in 15.459768881s (15.459859185s including waiting)
  Normal  Created  11m   kubelet  Created container nginxpersistent
  Normal  Started  11m   kubelet  Started container nginxpersistent

[jacy1101]

I found a similar answer.

Unable to create sandbox