I0428 13:57:10.763809 1 main.go:96] Version: v4.2.0 I0428 13:57:10.765736 1 connection.go:154] Connecting to unix:///csi/csi-provisioner.sock I0428 13:57:15.724367 1 connection.go:183] GRPC call: /csi.v1.Identity/GetPluginInfo I0428 13:57:15.724390 1 connection.go:184] GRPC request: {} I0428 13:57:15.729894 1 connection.go:186] GRPC response: {"name":"rbd.csi.ceph.com","vendor_version":"v3.5.1"} I0428 13:57:15.730006 1 connection.go:187] GRPC error: I0428 13:57:15.730025 1 main.go:154] CSI driver name: "rbd.csi.ceph.com" I0428 13:57:15.730036 1 common.go:111] Probing CSI driver for readiness I0428 13:57:15.730049 1 connection.go:183] GRPC call: /csi.v1.Identity/Probe I0428 13:57:15.730056 1 connection.go:184] GRPC request: {} I0428 13:57:15.730934 1 connection.go:186] GRPC response: {} I0428 13:57:15.731030 1 connection.go:187] GRPC error: I0428 13:57:15.731041 1 connection.go:183] GRPC call: /csi.v1.Controller/ControllerGetCapabilities I0428 13:57:15.731049 1 connection.go:184] GRPC request: {} I0428 13:57:15.732101 1 connection.go:186] GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":5}}},{"Type":{"Rpc":{"type":7}}},{"Type":{"Rpc":{"type":9}}}]} I0428 13:57:15.732316 1 connection.go:187] GRPC error: I0428 13:57:15.732363 1 main.go:192] Start NewCSISnapshotSideCarController with snapshotter [rbd.csi.ceph.com] kubeconfig [] csiTimeout [1m0s] csiAddress [unix:///csi/csi-provisioner.sock] resyncPeriod [15m0s] snapshotNamePrefix [snapshot] snapshotNameUUIDLength [824640405592] I0428 13:57:15.734939 1 leaderelection.go:248] attempting to acquire leader lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com... I0428 13:57:15.917316 1 leaderelection.go:258] successfully acquired lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:57:15.917532 1 leader_election.go:212] new leader detected, current leader: ceph-csi-rbd-provisioner-6f6497f79c-7x22v I0428 13:57:15.917598 1 leader_election.go:205] became leader, starting I0428 13:57:15.917878 1 snapshot_controller_base.go:115] Starting CSI snapshotter I0428 13:57:15.918115 1 reflector.go:219] Starting reflector *v1.VolumeSnapshotClass (15m0s) from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 I0428 13:57:15.918146 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 I0428 13:57:15.918326 1 reflector.go:219] Starting reflector *v1.VolumeSnapshotContent (15m0s) from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 I0428 13:57:15.918346 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:57:15.919807 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) E0428 13:57:15.920688 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 13:57:16.216832 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:57:17.254063 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:57:17.256232 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 13:57:17.473344 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:57:17.475985 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 13:57:19.557298 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:57:19.559234 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 13:57:19.755733 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:57:19.757652 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 13:57:21.226545 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:57:23.167853 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:57:23.169498 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 13:57:24.957685 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:57:24.959506 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 13:57:26.238420 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:57:30.571353 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:57:30.573255 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 13:57:31.250378 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:57:31.981201 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:57:31.982579 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 13:57:36.260643 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:57:41.268513 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:57:46.277952 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:57:47.225524 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:57:47.228119 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 13:57:51.287550 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:57:51.377931 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:57:51.380422 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 13:57:56.299627 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:58:01.309076 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:58:06.319163 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:58:11.327901 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:58:16.343551 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:58:21.357122 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:58:22.466136 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:58:22.468078 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 13:58:26.368274 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:58:31.380193 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:58:33.658126 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:58:33.660726 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 13:58:36.391331 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:58:41.401611 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:58:46.414330 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:58:51.423775 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:58:56.433321 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:59:01.441424 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:59:03.888946 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:59:03.890946 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 13:59:06.452912 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:59:11.465953 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:59:13.202861 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:59:13.204708 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 13:59:16.475863 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:59:21.484775 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:59:26.492100 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:59:31.501147 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:59:36.515861 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:59:41.525116 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:59:46.535776 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:59:47.958141 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:59:47.960531 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 13:59:51.547531 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 13:59:51.696810 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 13:59:51.700459 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 13:59:56.557694 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:00:01.568429 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:00:06.579235 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:00:11.590421 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:00:16.611357 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:00:21.660246 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:00:26.670397 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:00:26.754657 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 14:00:26.757221 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 14:00:31.680332 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:00:36.690593 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:00:41.699614 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:00:42.073850 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 14:00:42.075716 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 14:00:46.707883 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:00:51.715055 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:00:56.724328 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:01:01.737279 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:01:03.314595 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotContent from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 14:01:03.315993 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotContent: failed to list *v1.VolumeSnapshotContent: the server could not find the requested resource (get volumesnapshotcontents.snapshot.storage.k8s.io) I0428 14:01:06.744665 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:01:15.911163 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com I0428 14:01:18.171699 1 reflector.go:255] Listing and watching *v1.VolumeSnapshotClass from github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117 E0428 14:01:18.174023 1 reflector.go:138] github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions/factory.go:117: Failed to watch *v1.VolumeSnapshotClass: failed to list *v1.VolumeSnapshotClass: the server could not find the requested resource (get volumesnapshotclasses.snapshot.storage.k8s.io) I0428 14:01:24.440672 1 leaderelection.go:278] successfully renewed lease kube-system/external-snapshotter-leader-rbd-csi-ceph-com