{"level":"warn","ts":"2026-03-31T00:39:59.911765Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"warn","ts":"2026-03-31T00:39:59.911885Z","caller":"etcdmain/config.go:270","msg":"--snapshot-count is deprecated in 3.6 and will be decommissioned in 3.7."} {"level":"info","ts":"2026-03-31T00:39:59.911894Z","caller":"etcdmain/etcd.go:64","msg":"Running: ","args":["etcd","--advertise-client-urls=https://199.204.45.116:2379","--cert-file=/etc/kubernetes/pki/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/etcd","--feature-gates=InitialCorruptCheck=true","--initial-advertise-peer-urls=https://199.204.45.116:2380","--initial-cluster=controller-2=https://199.204.45.116:2380,controller-1=https://162.253.55.204:2380","--initial-cluster-state=existing","--key-file=/etc/kubernetes/pki/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://199.204.45.116:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://199.204.45.116:2380","--name=controller-2","--peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/etc/kubernetes/pki/etcd/peer.key","--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--snapshot-count=10000","--tls-min-version=TLS1.3","--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--watch-progress-notify-interval=5s"]} {"level":"info","ts":"2026-03-31T00:39:59.912032Z","caller":"etcdmain/etcd.go:124","msg":"Initialize and start etcd server","data-dir":"/var/lib/etcd","dir-type":"empty"} {"level":"warn","ts":"2026-03-31T00:39:59.912049Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-03-31T00:39:59.912059Z","caller":"embed/etcd.go:138","msg":"configuring peer listeners","listen-peer-urls":["https://199.204.45.116:2380"]} {"level":"info","ts":"2026-03-31T00:39:59.912612Z","caller":"embed/etcd.go:544","msg":"starting with peer TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-31T00:39:59.913019Z","caller":"embed/etcd.go:146","msg":"configuring client listeners","listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.116:2379"]} {"level":"info","ts":"2026-03-31T00:39:59.913113Z","caller":"embed/etcd.go:323","msg":"starting an etcd server","etcd-version":"3.6.4","git-sha":"5400cdc","go-version":"go1.23.11","go-os":"linux","go-arch":"amd64","max-cpu-set":8,"max-cpu-available":8,"member-initialized":false,"name":"controller-2","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://199.204.45.116:2380"],"listen-peer-urls":["https://199.204.45.116:2380"],"advertise-client-urls":["https://199.204.45.116:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.116:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"],"experimental-local-address":"","cors":["*"],"host-whitelist":["*"],"initial-cluster":"controller-1=https://162.253.55.204:2380,controller-2=https://199.204.45.116:2380","initial-cluster-state":"existing","initial-cluster-token":"etcd-cluster","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"feature-gates":"InitialCorruptCheck=true","initial-corrupt-check":false,"corrupt-check-time-interval":"0s","compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","discovery-token":"","discovery-endpoints":"","discovery-dial-timeout":"2s","discovery-request-timeout":"5s","discovery-keepalive-time":"2s","discovery-keepalive-timeout":"6s","discovery-insecure-transport":true,"discovery-insecure-skip-tls-verify":false,"discovery-cert":"","discovery-key":"","discovery-cacert":"","discovery-user":"","downgrade-check-interval":"5s","max-learners":1,"v2-deprecation":"write-only"} {"level":"info","ts":"2026-03-31T00:39:59.914633Z","logger":"bbolt","caller":"backend/backend.go:203","msg":"Opening db file (/var/lib/etcd/member/snap/db) with mode -rw------- and with options: {Timeout: 0s, NoGrowSync: false, NoFreelistSync: true, PreLoadFreelist: false, FreelistType: hashmap, ReadOnly: false, MmapFlags: 8000, InitialMmapSize: 10737418240, PageSize: 0, NoSync: false, OpenFile: 0x0, Mlock: false, Logger: 0xc00041e120}"} {"level":"info","ts":"2026-03-31T00:39:59.919130Z","logger":"bbolt","caller":"bbolt@v1.4.2/db.go:321","msg":"Opening bbolt db (/var/lib/etcd/member/snap/db) successfully"} {"level":"info","ts":"2026-03-31T00:39:59.919184Z","caller":"storage/backend.go:80","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"4.668655ms"} {"level":"info","ts":"2026-03-31T00:39:59.919230Z","caller":"etcdserver/bootstrap.go:220","msg":"restore consistentIndex","index":0} {"level":"info","ts":"2026-03-31T00:39:59.919244Z","caller":"etcdserver/bootstrap.go:94","msg":"bootstrapping cluster"} {"level":"info","ts":"2026-03-31T00:39:59.930121Z","caller":"netutil/netutil.go:113","msg":"resolved URL Host","url":"https://162.253.55.204:2380","host":"162.253.55.204:2380","resolved-addr":"162.253.55.204:2380"} {"level":"info","ts":"2026-03-31T00:39:59.930172Z","caller":"netutil/netutil.go:113","msg":"resolved URL Host","url":"https://199.204.45.116:2380","host":"199.204.45.116:2380","resolved-addr":"199.204.45.116:2380"} {"level":"info","ts":"2026-03-31T00:39:59.931561Z","caller":"etcdserver/bootstrap.go:101","msg":"bootstrapping storage"} {"level":"info","ts":"2026-03-31T00:39:59.945156Z","caller":"etcdserver/bootstrap.go:109","msg":"bootstrapping raft"} {"level":"info","ts":"2026-03-31T00:39:59.945191Z","caller":"etcdserver/bootstrap.go:499","msg":"starting local member","local-member-id":"3119206cae914f30","cluster-id":"97da89d0cbebd2b5"} {"level":"info","ts":"2026-03-31T00:39:59.945212Z","caller":"etcdserver/server.go:312","msg":"bootstrap successfully"} {"level":"info","ts":"2026-03-31T00:39:59.945280Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"3119206cae914f30 switched to configuration voters=()"} {"level":"info","ts":"2026-03-31T00:39:59.945321Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"3119206cae914f30 became follower at term 0"} {"level":"info","ts":"2026-03-31T00:39:59.945331Z","logger":"raft","caller":"v3@v3.6.0/raft.go:493","msg":"newRaft 3119206cae914f30 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]"} {"level":"warn","ts":"2026-03-31T00:39:59.954521Z","caller":"auth/store.go:1135","msg":"simple token is not cryptographically signed"} {"level":"info","ts":"2026-03-31T00:39:59.955722Z","caller":"mvcc/kvstore.go:408","msg":"kvstore restored","current-rev":1} {"level":"info","ts":"2026-03-31T00:39:59.958015Z","caller":"storage/quota.go:93","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"} {"level":"info","ts":"2026-03-31T00:39:59.958462Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.958483Z","caller":"rafthttp/transport.go:286","msg":"added new remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7","remote-peer-urls":["https://162.253.55.204:2380"]} {"level":"info","ts":"2026-03-31T00:39:59.958497Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.958509Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.959288Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.960532Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.960608Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7","remote-peer-urls":["https://162.253.55.204:2380"]} {"level":"info","ts":"2026-03-31T00:39:59.960549Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.960666Z","caller":"etcdserver/server.go:598","msg":"starting etcd server","local-member-id":"3119206cae914f30","local-server-version":"3.6.4","cluster-version":"to_be_decided"} {"level":"info","ts":"2026-03-31T00:39:59.960662Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.960726Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.960940Z","caller":"embed/etcd.go:766","msg":"starting with client TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/server.crt, key = /etc/kubernetes/pki/etcd/server.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-31T00:39:59.961256Z","caller":"etcdserver/server.go:499","msg":"starting initial election tick advance","election-ticks":10} {"level":"info","ts":"2026-03-31T00:39:59.961267Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-31T00:39:59.961326Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-31T00:39:59.961348Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-31T00:39:59.961390Z","caller":"embed/etcd.go:292","msg":"now serving peer/client/metrics","local-member-id":"3119206cae914f30","initial-advertise-peer-urls":["https://199.204.45.116:2380"],"listen-peer-urls":["https://199.204.45.116:2380"],"advertise-client-urls":["https://199.204.45.116:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.204.45.116:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"]} {"level":"info","ts":"2026-03-31T00:39:59.961441Z","caller":"embed/etcd.go:890","msg":"serving metrics","address":"http://127.0.0.1:2381"} {"level":"info","ts":"2026-03-31T00:39:59.961549Z","caller":"embed/etcd.go:640","msg":"serving peer traffic","address":"199.204.45.116:2380"} {"level":"info","ts":"2026-03-31T00:39:59.961569Z","caller":"embed/etcd.go:611","msg":"cmux::serve","address":"199.204.45.116:2380"} {"level":"info","ts":"2026-03-31T00:39:59.961901Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"3119206cae914f30","to":"a11c20306510c1f7","stream-type":"stream Message"} {"level":"info","ts":"2026-03-31T00:39:59.961921Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.961987Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.963693Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"3119206cae914f30","to":"a11c20306510c1f7","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-31T00:39:59.963718Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.978531Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.978713Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:39:59.979912Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1116","msg":"3119206cae914f30 [term: 0] received a MsgHeartbeat message with higher term from a11c20306510c1f7 [term: 2]"} {"level":"info","ts":"2026-03-31T00:39:59.979948Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"3119206cae914f30 became follower at term 2"} {"level":"info","ts":"2026-03-31T00:39:59.979963Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: 3119206cae914f30 elected leader a11c20306510c1f7 at term 2"} {"level":"info","ts":"2026-03-31T00:40:00.039327Z","caller":"rafthttp/http.go:258","msg":"receiving database snapshot","local-member-id":"3119206cae914f30","remote-snapshot-sender-id":"a11c20306510c1f7","incoming-snapshot-index":562,"incoming-snapshot-message-size-bytes":8201,"incoming-snapshot-message-size":"8.2 kB"} {"level":"info","ts":"2026-03-31T00:40:00.054556Z","caller":"snap/db.go:65","msg":"saved database snapshot to disk","path":"/var/lib/etcd/member/snap/0000000000000232.snap.db","bytes":843776,"size":"844 kB"} {"level":"info","ts":"2026-03-31T00:40:00.054632Z","caller":"rafthttp/http.go:287","msg":"received and saved database snapshot","local-member-id":"3119206cae914f30","remote-snapshot-sender-id":"a11c20306510c1f7","incoming-snapshot-index":562,"incoming-snapshot-size-bytes":843776,"incoming-snapshot-size":"844 kB","download-took":"15.355887ms"} {"level":"info","ts":"2026-03-31T00:40:00.054946Z","logger":"raft","caller":"v3@v3.6.0/log.go:465","msg":"log [committed=0, applied=0, applying=0, unstable.offset=1, unstable.offsetInProgress=1, len(unstable.Entries)=0] starts to restore snapshot [index: 562, term: 2]"} {"level":"info","ts":"2026-03-31T00:40:00.055288Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"3119206cae914f30 switched to configuration voters=(11609189331680412151) learners=(3537894633433550640)"} {"level":"info","ts":"2026-03-31T00:40:00.055521Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1935","msg":"3119206cae914f30 [commit: 562, lastindex: 562, lastterm: 2] restored snapshot [index: 562, term: 2]"} {"level":"info","ts":"2026-03-31T00:40:00.055618Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1844","msg":"3119206cae914f30 [commit: 562] restored snapshot [index: 562, term: 2]"} {"level":"info","ts":"2026-03-31T00:40:00.055854Z","caller":"etcdserver/server.go:1007","msg":"applying snapshot","current-snapshot-index":0,"current-applied-index":0,"incoming-leader-snapshot-index":562,"incoming-leader-snapshot-term":2} {"level":"info","ts":"2026-03-31T00:40:00.063816Z","caller":"etcdserver/raft.go:276","msg":"applied incoming Raft snapshot","snapshot-index":562} {"level":"info","ts":"2026-03-31T00:40:00.064412Z","logger":"bbolt","caller":"backend/backend.go:203","msg":"Opening db file (/var/lib/etcd/member/snap/db) with mode -rw------- and with options: {Timeout: 0s, NoGrowSync: false, NoFreelistSync: true, PreLoadFreelist: false, FreelistType: hashmap, ReadOnly: false, MmapFlags: 8000, InitialMmapSize: 10737418240, PageSize: 0, NoSync: false, OpenFile: 0x0, Mlock: false, Logger: 0xc000136120}"} {"level":"info","ts":"2026-03-31T00:40:00.065274Z","logger":"bbolt","caller":"bbolt@v1.4.2/db.go:321","msg":"Opening bbolt db (/var/lib/etcd/member/snap/db) successfully"} {"level":"info","ts":"2026-03-31T00:40:00.065339Z","caller":"storage/backend.go:80","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"1.043479ms"} {"level":"info","ts":"2026-03-31T00:40:00.065406Z","caller":"etcdserver/server.go:1055","msg":"restoring lease store"} {"level":"info","ts":"2026-03-31T00:40:00.069185Z","caller":"etcdserver/server.go:1059","msg":"restored lease store"} {"level":"info","ts":"2026-03-31T00:40:00.069247Z","caller":"etcdserver/server.go:1062","msg":"restoring mvcc store"} {"level":"info","ts":"2026-03-31T00:40:00.071915Z","caller":"mvcc/kvstore.go:408","msg":"kvstore restored","current-rev":532} {"level":"info","ts":"2026-03-31T00:40:00.072003Z","caller":"etcdserver/server.go:1070","msg":"restored mvcc store","consistent-index":562} {"level":"info","ts":"2026-03-31T00:40:00.072027Z","caller":"etcdserver/server.go:1090","msg":"restoring alarm store"} {"level":"info","ts":"2026-03-31T00:40:00.072119Z","caller":"etcdserver/server.go:1078","msg":"closing old backend file"} {"level":"info","ts":"2026-03-31T00:40:00.073685Z","caller":"etcdserver/server.go:1096","msg":"restored alarm store"} {"level":"info","ts":"2026-03-31T00:40:00.073689Z","caller":"etcdserver/server.go:1080","msg":"closed old backend file"} {"level":"info","ts":"2026-03-31T00:40:00.073714Z","caller":"etcdserver/server.go:1099","msg":"restoring auth store"} {"level":"info","ts":"2026-03-31T00:40:00.073759Z","caller":"etcdserver/server.go:1103","msg":"restored auth store"} {"level":"info","ts":"2026-03-31T00:40:00.073777Z","caller":"etcdserver/server.go:1106","msg":"restoring v2 store"} {"level":"info","ts":"2026-03-31T00:40:00.074992Z","caller":"etcdserver/server.go:1115","msg":"restored v2 store"} {"level":"info","ts":"2026-03-31T00:40:00.075032Z","caller":"etcdserver/server.go:1119","msg":"restoring cluster configuration"} {"level":"info","ts":"2026-03-31T00:40:00.075093Z","caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.6"} {"level":"info","ts":"2026-03-31T00:40:00.075129Z","caller":"membership/cluster.go:297","msg":"recovered/added member from store","cluster-id":"97da89d0cbebd2b5","local-member-id":"3119206cae914f30","recovered-remote-peer-id":"3119206cae914f30","recovered-remote-peer-urls":["https://199.204.45.116:2380"],"recovered-remote-peer-is-learner":true} {"level":"info","ts":"2026-03-31T00:40:00.075143Z","caller":"membership/cluster.go:297","msg":"recovered/added member from store","cluster-id":"97da89d0cbebd2b5","local-member-id":"3119206cae914f30","recovered-remote-peer-id":"a11c20306510c1f7","recovered-remote-peer-urls":["https://162.253.55.204:2380"],"recovered-remote-peer-is-learner":false} {"level":"info","ts":"2026-03-31T00:40:00.075151Z","caller":"membership/cluster.go:307","msg":"set cluster version from store","cluster-version":"3.6"} {"level":"info","ts":"2026-03-31T00:40:00.075159Z","caller":"etcdserver/server.go:1123","msg":"restored cluster configuration"} {"level":"info","ts":"2026-03-31T00:40:00.075168Z","caller":"etcdserver/server.go:1124","msg":"removing old peers from network"} {"level":"info","ts":"2026-03-31T00:40:00.075179Z","caller":"rafthttp/peer.go:316","msg":"stopping remote peer","remote-peer-id":"a11c20306510c1f7"} {"level":"warn","ts":"2026-03-31T00:40:00.075249Z","caller":"rafthttp/stream.go:285","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.075271Z","caller":"rafthttp/stream.go:293","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"a11c20306510c1f7"} {"level":"warn","ts":"2026-03-31T00:40:00.075302Z","caller":"rafthttp/stream.go:285","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.075315Z","caller":"rafthttp/stream.go:293","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.075413Z","caller":"rafthttp/pipeline.go:85","msg":"stopped HTTP pipelining with remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"warn","ts":"2026-03-31T00:40:00.076053Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7","error":"context canceled"} {"level":"warn","ts":"2026-03-31T00:40:00.076292Z","caller":"rafthttp/peer_status.go:66","msg":"peer became inactive (message send to peer failed)","peer-id":"a11c20306510c1f7","error":"failed to read a11c20306510c1f7 on stream MsgApp v2 (context canceled)"} {"level":"info","ts":"2026-03-31T00:40:00.076478Z","caller":"rafthttp/stream.go:441","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"warn","ts":"2026-03-31T00:40:00.076790Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7","error":"context canceled"} {"level":"info","ts":"2026-03-31T00:40:00.076926Z","caller":"rafthttp/stream.go:441","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.077078Z","caller":"rafthttp/peer.go:321","msg":"stopped remote peer","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.077284Z","caller":"rafthttp/transport.go:354","msg":"removed remote peer","local-member-id":"3119206cae914f30","removed-remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.077360Z","caller":"etcdserver/server.go:1129","msg":"removed old peers from network"} {"level":"info","ts":"2026-03-31T00:40:00.077609Z","caller":"etcdserver/server.go:1130","msg":"adding peers from new cluster configuration"} {"level":"info","ts":"2026-03-31T00:40:00.077817Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.077949Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.079735Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.079862Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.084996Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.085234Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7","remote-peer-urls":["https://162.253.55.204:2380"]} {"level":"info","ts":"2026-03-31T00:40:00.085154Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.085290Z","caller":"etcdserver/server.go:1139","msg":"added peers from new cluster configuration"} {"level":"info","ts":"2026-03-31T00:40:00.085460Z","caller":"etcdserver/server.go:1015","msg":"applied snapshot","current-snapshot-index":562,"current-applied-index":562,"incoming-leader-snapshot-index":562,"incoming-leader-snapshot-term":2} {"level":"info","ts":"2026-03-31T00:40:00.085122Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.089060Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"3119206cae914f30","to":"a11c20306510c1f7","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-31T00:40:00.089097Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.089133Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.090987Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"3119206cae914f30","to":"a11c20306510c1f7","stream-type":"stream Message"} {"level":"info","ts":"2026-03-31T00:40:00.091020Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.094808Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.097315Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"a11c20306510c1f7"} {"level":"info","ts":"2026-03-31T00:40:00.437922Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"3119206cae914f30 switched to configuration voters=(3537894633433550640 11609189331680412151)"} {"level":"info","ts":"2026-03-31T00:40:00.438340Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"97da89d0cbebd2b5","local-member-id":"3119206cae914f30","promoted-member-id":"3119206cae914f30"} {"level":"warn","ts":"2026-03-31T00:40:05.817073Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"504.297548ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-31T00:40:06.962105Z","caller":"etcdserver/server.go:1814","msg":"failed to publish local member to cluster through raft","local-member-id":"3119206cae914f30","local-member-attributes":"{Name:controller-2 ClientURLs:[https://199.204.45.116:2379]}","publish-timeout":"7s","error":"etcdserver: request timed out, possibly due to connection lost"} {"level":"info","ts":"2026-03-31T00:40:06.972802Z","caller":"etcdserver/server.go:1804","msg":"published local member to cluster through raft","local-member-id":"3119206cae914f30","local-member-attributes":"{Name:controller-2 ClientURLs:[https://199.204.45.116:2379]}","cluster-id":"97da89d0cbebd2b5","publish-timeout":"7s"} {"level":"info","ts":"2026-03-31T00:40:06.972837Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-31T00:40:06.972913Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-31T00:40:06.973731Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} {"level":"info","ts":"2026-03-31T00:40:06.973783Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} {"level":"warn","ts":"2026-03-31T00:40:06.974565Z","caller":"v3rpc/grpc.go:52","msg":"etcdserver: failed to register grpc metrics","error":"duplicate metrics collector registration attempted"} {"level":"info","ts":"2026-03-31T00:40:06.974608Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-31T00:40:06.974748Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-31T00:40:06.979761Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"199.204.45.116:2379"} {"level":"info","ts":"2026-03-31T00:40:06.985769Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"127.0.0.1:2379"} {"level":"warn","ts":"2026-03-31T00:40:06.986006Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.116:50964","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:06.987293Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.116:50936","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:06.987747Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.116:50932","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:06.988170Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48520","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:06.988840Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48538","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:06.990739Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.116:50988","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:06.990938Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.116:50916","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:06.991859Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.116:50978","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:06.991963Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.116:50946","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:06.989084Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.116:50926","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:06.990541Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.116:50950","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:06.991956Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48560","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.000705Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48612","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.014367Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48650","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.026994Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48678","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.034397Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48708","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.060537Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48744","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.068436Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48768","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.078198Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48796","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.088564Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48802","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.106852Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48834","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.116982Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48854","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.125036Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48882","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.133874Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48890","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.141639Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48920","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.152782Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48930","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.160632Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48954","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.169559Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:48966","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.177434Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49008","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.220132Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49012","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.242661Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49046","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.254552Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49060","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.262355Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49084","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.273751Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49104","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.281448Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49132","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.291915Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49146","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.300774Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49158","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.313280Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49176","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.323599Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49182","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.334736Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49188","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.343536Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49214","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.353072Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49226","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.363814Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49244","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.375046Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49258","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.383259Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49274","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.401020Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49296","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.448256Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49324","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.459844Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49334","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.471271Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49346","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.480484Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49358","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.489344Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49374","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.497904Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49380","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.507585Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49392","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.518213Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49400","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.528699Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49426","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.536860Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49458","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.547138Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49472","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.556585Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49496","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.564331Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49524","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.573596Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49540","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.594260Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49554","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.602988Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49572","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.612840Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:49592","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.621145Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:53296","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.657965Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:53322","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.668892Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:53330","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.678987Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:53350","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:07.739600Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:53374","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-31T00:40:11.524722Z","caller":"traceutil/trace.go:172","msg":"trace[599101263] linearizableReadLoop","detail":"{readStateIndex:595; appliedIndex:596; }","duration":"170.394697ms","start":"2026-03-31T00:40:11.354294Z","end":"2026-03-31T00:40:11.524689Z","steps":["trace[599101263] 'read index received' (duration: 170.388437ms)","trace[599101263] 'applied index is now lower than readState.Index' (duration: 5.02µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:11.524982Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"170.649006ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:481"} {"level":"info","ts":"2026-03-31T00:40:11.525097Z","caller":"traceutil/trace.go:172","msg":"trace[2027717405] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:561; }","duration":"170.804271ms","start":"2026-03-31T00:40:11.354273Z","end":"2026-03-31T00:40:11.525077Z","steps":["trace[2027717405] 'agreement among raft nodes before linearized reading' (duration: 170.501811ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:12.713717Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807293976,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:40:12.944743Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.076880178s","expected-duration":"1s"} {"level":"info","ts":"2026-03-31T00:40:12.948263Z","caller":"traceutil/trace.go:172","msg":"trace[990334094] linearizableReadLoop","detail":"{readStateIndex:597; appliedIndex:597; }","duration":"734.969949ms","start":"2026-03-31T00:40:12.213265Z","end":"2026-03-31T00:40:12.948235Z","steps":["trace[990334094] 'read index received' (duration: 734.964979ms)","trace[990334094] 'applied index is now lower than readState.Index' (duration: 4.09µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:12.948470Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"735.184606ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:12.948552Z","caller":"traceutil/trace.go:172","msg":"trace[227225149] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:562; }","duration":"735.279199ms","start":"2026-03-31T00:40:12.213251Z","end":"2026-03-31T00:40:12.948530Z","steps":["trace[227225149] 'agreement among raft nodes before linearized reading' (duration: 735.124474ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:12.949019Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:12.213229Z","time spent":"735.591809ms","remote":"127.0.0.1:48522","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-31T00:40:12.950147Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"360.573682ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-31T00:40:12.950219Z","caller":"traceutil/trace.go:172","msg":"trace[992122725] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:562; }","duration":"360.665985ms","start":"2026-03-31T00:40:12.589539Z","end":"2026-03-31T00:40:12.950205Z","steps":["trace[992122725] 'agreement among raft nodes before linearized reading' (duration: 360.516ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:12.950259Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:12.589515Z","time spent":"360.734107ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-03-31T00:40:13.143058Z","caller":"traceutil/trace.go:172","msg":"trace[452290752] linearizableReadLoop","detail":"{readStateIndex:598; appliedIndex:597; }","duration":"185.628238ms","start":"2026-03-31T00:40:12.957405Z","end":"2026-03-31T00:40:13.143034Z","steps":["trace[452290752] 'read index received' (duration: 120.019042ms)","trace[452290752] 'applied index is now lower than readState.Index' (duration: 65.608016ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:13.143211Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"185.782803ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:13.143251Z","caller":"traceutil/trace.go:172","msg":"trace[604367026] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:563; }","duration":"185.837815ms","start":"2026-03-31T00:40:12.957401Z","end":"2026-03-31T00:40:13.143239Z","steps":["trace[604367026] 'agreement among raft nodes before linearized reading' (duration: 185.748202ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:40:15.061939Z","caller":"traceutil/trace.go:172","msg":"trace[1126625085] linearizableReadLoop","detail":"{readStateIndex:604; appliedIndex:604; }","duration":"101.564591ms","start":"2026-03-31T00:40:14.960350Z","end":"2026-03-31T00:40:15.061915Z","steps":["trace[1126625085] 'read index received' (duration: 101.55762ms)","trace[1126625085] 'applied index is now lower than readState.Index' (duration: 5.811µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:15.062084Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"101.719016ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:15.062131Z","caller":"traceutil/trace.go:172","msg":"trace[313186293] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:568; }","duration":"101.776688ms","start":"2026-03-31T00:40:14.960342Z","end":"2026-03-31T00:40:15.062119Z","steps":["trace[313186293] 'agreement among raft nodes before linearized reading' (duration: 101.685805ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:15.617147Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"176.48856ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:15.617257Z","caller":"traceutil/trace.go:172","msg":"trace[884032301] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:570; }","duration":"176.682796ms","start":"2026-03-31T00:40:15.440554Z","end":"2026-03-31T00:40:15.617237Z","steps":["trace[884032301] 'agreement among raft nodes before linearized reading' (duration: 45.257385ms)","trace[884032301] 'range keys from in-memory index tree' (duration: 131.202324ms)"],"step_count":2} {"level":"info","ts":"2026-03-31T00:40:16.562084Z","caller":"traceutil/trace.go:172","msg":"trace[797058876] linearizableReadLoop","detail":"{readStateIndex:609; appliedIndex:608; }","duration":"239.535022ms","start":"2026-03-31T00:40:16.322518Z","end":"2026-03-31T00:40:16.562053Z","steps":["trace[797058876] 'read index received' (duration: 156.877825ms)","trace[797058876] 'applied index is now lower than readState.Index' (duration: 82.656467ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:16.562366Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"239.774149ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-31T00:40:16.562541Z","caller":"traceutil/trace.go:172","msg":"trace[1474346527] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:573; }","duration":"240.003916ms","start":"2026-03-31T00:40:16.322506Z","end":"2026-03-31T00:40:16.562510Z","steps":["trace[1474346527] 'agreement among raft nodes before linearized reading' (duration: 239.644295ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:16.564354Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"172.462516ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-31T00:40:16.564413Z","caller":"traceutil/trace.go:172","msg":"trace[447530016] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:574; }","duration":"172.545978ms","start":"2026-03-31T00:40:16.391853Z","end":"2026-03-31T00:40:16.564399Z","steps":["trace[447530016] 'agreement among raft nodes before linearized reading' (duration: 172.28599ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:17.542967Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"319.153889ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-31T00:40:17.543545Z","caller":"traceutil/trace.go:172","msg":"trace[1791847345] linearizableReadLoop","detail":"{readStateIndex:613; appliedIndex:611; }","duration":"297.466894ms","start":"2026-03-31T00:40:17.246059Z","end":"2026-03-31T00:40:17.543526Z","steps":["trace[1791847345] 'read index received' (duration: 234.717924ms)","trace[1791847345] 'applied index is now lower than readState.Index' (duration: 62.74746ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:17.544192Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"297.883848ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csinodes/controller-2\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:17.544396Z","caller":"traceutil/trace.go:172","msg":"trace[350631558] range","detail":"{range_begin:/registry/csinodes/controller-2; range_end:; response_count:0; response_revision:577; }","duration":"298.334082ms","start":"2026-03-31T00:40:17.246045Z","end":"2026-03-31T00:40:17.544380Z","steps":["trace[350631558] 'agreement among raft nodes before linearized reading' (duration: 297.748734ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:17.588187Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"123.580393ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:440"} {"level":"info","ts":"2026-03-31T00:40:17.588239Z","caller":"traceutil/trace.go:172","msg":"trace[2073950522] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:578; }","duration":"123.640015ms","start":"2026-03-31T00:40:17.464590Z","end":"2026-03-31T00:40:17.588230Z","steps":["trace[2073950522] 'agreement among raft nodes before linearized reading' (duration: 123.553482ms)"],"step_count":1} 2026/03/31 00:40:17 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-31T00:40:18.167115Z","caller":"traceutil/trace.go:172","msg":"trace[662255537] linearizableReadLoop","detail":"{readStateIndex:615; appliedIndex:615; }","duration":"481.168076ms","start":"2026-03-31T00:40:17.685921Z","end":"2026-03-31T00:40:18.167089Z","steps":["trace[662255537] 'read index received' (duration: 481.162706ms)","trace[662255537] 'applied index is now lower than readState.Index' (duration: 4.36µs)"],"step_count":2} {"level":"info","ts":"2026-03-31T00:40:18.167273Z","caller":"traceutil/trace.go:172","msg":"trace[1577304821] transaction","detail":"{read_only:false; response_revision:580; number_of_response:1; }","duration":"347.958588ms","start":"2026-03-31T00:40:17.819294Z","end":"2026-03-31T00:40:18.167252Z","steps":["trace[1577304821] 'process raft request' (duration: 347.889926ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:18.167404Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:17.819275Z","time spent":"348.070932ms","remote":"127.0.0.1:48914","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":3009,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-31T00:40:18.168952Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"357.268862ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-node-lease/controller-2\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:18.169042Z","caller":"traceutil/trace.go:172","msg":"trace[549322455] range","detail":"{range_begin:/registry/leases/kube-node-lease/controller-2; range_end:; response_count:0; response_revision:580; }","duration":"357.371186ms","start":"2026-03-31T00:40:17.811652Z","end":"2026-03-31T00:40:18.169023Z","steps":["trace[549322455] 'agreement among raft nodes before linearized reading' (duration: 357.225421ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:18.169103Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:17.811610Z","time spent":"357.47646ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":49,"response count":0,"response size":28,"request content":"key:\"/registry/leases/kube-node-lease/controller-2\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:40:18.169153Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"208.403387ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:18.169283Z","caller":"traceutil/trace.go:172","msg":"trace[492351122] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:580; }","duration":"208.583192ms","start":"2026-03-31T00:40:17.960672Z","end":"2026-03-31T00:40:18.169255Z","steps":["trace[492351122] 'agreement among raft nodes before linearized reading' (duration: 208.280943ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:18.532652Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"235.838502ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-31T00:40:18.579597Z","caller":"traceutil/trace.go:172","msg":"trace[1728558208] transaction","detail":"{read_only:false; response_revision:582; number_of_response:1; }","duration":"401.930262ms","start":"2026-03-31T00:40:18.177634Z","end":"2026-03-31T00:40:18.579564Z","steps":["trace[1728558208] 'process raft request' (duration: 401.784087ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:18.579794Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:18.177611Z","time spent":"402.092127ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":536,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-31T00:40:18.581157Z","caller":"traceutil/trace.go:172","msg":"trace[1861975450] transaction","detail":"{read_only:false; number_of_response:1; response_revision:582; }","duration":"398.937283ms","start":"2026-03-31T00:40:18.182193Z","end":"2026-03-31T00:40:18.581131Z","steps":["trace[1861975450] 'process raft request' (duration: 398.695355ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:40:18.582558Z","caller":"traceutil/trace.go:172","msg":"trace[363197849] linearizableReadLoop","detail":"{readStateIndex:618; appliedIndex:620; }","duration":"281.269578ms","start":"2026-03-31T00:40:18.301271Z","end":"2026-03-31T00:40:18.582541Z","steps":["trace[363197849] 'read index received' (duration: 281.263588ms)","trace[363197849] 'applied index is now lower than readState.Index' (duration: 4.96µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:18.582822Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"281.495164ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-31T00:40:18.582876Z","caller":"traceutil/trace.go:172","msg":"trace[469967432] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:583; }","duration":"281.607517ms","start":"2026-03-31T00:40:18.301252Z","end":"2026-03-31T00:40:18.582859Z","steps":["trace[469967432] 'agreement among raft nodes before linearized reading' (duration: 281.35835ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:18.583568Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:18.182168Z","time spent":"399.560531ms","remote":"127.0.0.1:48914","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":34,"response count":0,"response size":3230,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-31T00:40:19.120401Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294017,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-31T00:40:19.258704Z","caller":"traceutil/trace.go:172","msg":"trace[461849040] linearizableReadLoop","detail":"{readStateIndex:621; appliedIndex:624; }","duration":"639.414215ms","start":"2026-03-31T00:40:18.619262Z","end":"2026-03-31T00:40:19.258677Z","steps":["trace[461849040] 'read index received' (duration: 639.404505ms)","trace[461849040] 'applied index is now lower than readState.Index' (duration: 8.28µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:19.260239Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":5706233631807294020,"received-request-id":5706233631807294017} {"level":"warn","ts":"2026-03-31T00:40:19.639715Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.020420481s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:19.639806Z","caller":"traceutil/trace.go:172","msg":"trace[136521380] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:586; }","duration":"1.020530575s","start":"2026-03-31T00:40:18.619255Z","end":"2026-03-31T00:40:19.639785Z","steps":["trace[136521380] 'agreement among raft nodes before linearized reading' (duration: 639.515369ms)","trace[136521380] 'range keys from in-memory index tree' (duration: 380.876231ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:19.639790Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"414.480029ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-31T00:40:19.834184Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"193.293183ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:19.834255Z","caller":"traceutil/trace.go:172","msg":"trace[805425172] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:587; }","duration":"193.377556ms","start":"2026-03-31T00:40:19.640861Z","end":"2026-03-31T00:40:19.834238Z","steps":["trace[805425172] 'range keys from in-memory index tree' (duration: 193.215421ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:19.834259Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"194.301123ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-31T00:40:19.834586Z","caller":"traceutil/trace.go:172","msg":"trace[1474885151] linearizableReadLoop","detail":"{readStateIndex:629; appliedIndex:624; }","duration":"575.801824ms","start":"2026-03-31T00:40:19.258772Z","end":"2026-03-31T00:40:19.834574Z","steps":["trace[1474885151] 'read index received' (duration: 2.134163ms)","trace[1474885151] 'applied index is now lower than readState.Index' (duration: 573.666961ms)"],"step_count":2} {"level":"info","ts":"2026-03-31T00:40:19.834646Z","caller":"traceutil/trace.go:172","msg":"trace[874995052] transaction","detail":"{read_only:false; response_revision:588; number_of_response:1; }","duration":"1.188585412s","start":"2026-03-31T00:40:18.646036Z","end":"2026-03-31T00:40:19.834622Z","steps":["trace[874995052] 'process raft request' (duration: 1.188391126s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:19.834786Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:18.645997Z","time spent":"1.188711276s","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-31T00:40:19.834823Z","caller":"traceutil/trace.go:172","msg":"trace[1769823353] transaction","detail":"{read_only:false; number_of_response:1; response_revision:587; }","duration":"1.245770944s","start":"2026-03-31T00:40:18.589030Z","end":"2026-03-31T00:40:19.834801Z","steps":["trace[1769823353] 'process raft request' (duration: 1.245322901s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:19.834916Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:18.589000Z","time spent":"1.245869117s","remote":"127.0.0.1:48914","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":34,"response count":0,"response size":3604,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-31T00:40:19.834935Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.078344634s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csinodes/controller-2\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:19.835008Z","caller":"traceutil/trace.go:172","msg":"trace[1196405505] range","detail":"{range_begin:/registry/csinodes/controller-2; range_end:; response_count:0; response_revision:590; }","duration":"1.078389185s","start":"2026-03-31T00:40:18.756576Z","end":"2026-03-31T00:40:19.834965Z","steps":["trace[1196405505] 'agreement among raft nodes before linearized reading' (duration: 1.078076086s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:19.835046Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:18.756549Z","time spent":"1.078484478s","remote":"127.0.0.1:49372","response type":"/etcdserverpb.KV/Range","request count":0,"request size":35,"response count":0,"response size":28,"request content":"key:\"/registry/csinodes/controller-2\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:40:19.837487Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"399.890504ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-31T00:40:19.837598Z","caller":"traceutil/trace.go:172","msg":"trace[1643148544] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:592; }","duration":"400.021688ms","start":"2026-03-31T00:40:19.437554Z","end":"2026-03-31T00:40:19.837576Z","steps":["trace[1643148544] 'agreement among raft nodes before linearized reading' (duration: 399.795151ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:19.837649Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:19.437518Z","time spent":"400.122211ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:40:19.837791Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"187.940873ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-31T00:40:19.838319Z","caller":"traceutil/trace.go:172","msg":"trace[718438750] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:592; }","duration":"188.480869ms","start":"2026-03-31T00:40:19.649816Z","end":"2026-03-31T00:40:19.838296Z","steps":["trace[718438750] 'agreement among raft nodes before linearized reading' (duration: 187.818479ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:20.460608Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294026,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:40:20.581346Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"382.884833ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-31T00:40:20.581498Z","caller":"traceutil/trace.go:172","msg":"trace[1015821669] transaction","detail":"{read_only:false; response_revision:593; number_of_response:1; }","duration":"740.57629ms","start":"2026-03-31T00:40:19.840898Z","end":"2026-03-31T00:40:20.581474Z","steps":["trace[1015821669] 'process raft request' (duration: 357.46102ms)","trace[1015821669] 'compare' (duration: 382.740589ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:20.581570Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:19.840880Z","time spent":"740.661912ms","remote":"127.0.0.1:48914","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":3875,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-31T00:40:20.668134Z","caller":"traceutil/trace.go:172","msg":"trace[1951833114] transaction","detail":"{read_only:false; response_revision:595; number_of_response:1; }","duration":"792.837067ms","start":"2026-03-31T00:40:19.875283Z","end":"2026-03-31T00:40:20.668120Z","steps":["trace[1951833114] 'process raft request' (duration: 792.733094ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:20.668247Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:19.875245Z","time spent":"792.953261ms","remote":"127.0.0.1:48618","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":120,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-31T00:40:20.669165Z","caller":"traceutil/trace.go:172","msg":"trace[1497592289] linearizableReadLoop","detail":"{readStateIndex:634; appliedIndex:636; }","duration":"709.19596ms","start":"2026-03-31T00:40:19.959941Z","end":"2026-03-31T00:40:20.669137Z","steps":["trace[1497592289] 'read index received' (duration: 709.18997ms)","trace[1497592289] 'applied index is now lower than readState.Index' (duration: 4.85µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:20.669322Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"709.354735ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:20.669411Z","caller":"traceutil/trace.go:172","msg":"trace[1878446663] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:595; }","duration":"709.458618ms","start":"2026-03-31T00:40:19.959935Z","end":"2026-03-31T00:40:20.669394Z","steps":["trace[1878446663] 'agreement among raft nodes before linearized reading' (duration: 709.275772ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:40:20.669424Z","caller":"traceutil/trace.go:172","msg":"trace[2070300055] transaction","detail":"{read_only:false; response_revision:596; number_of_response:1; }","duration":"792.103414ms","start":"2026-03-31T00:40:19.877309Z","end":"2026-03-31T00:40:20.669413Z","steps":["trace[2070300055] 'process raft request' (duration: 792.008781ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:20.669475Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:19.959923Z","time spent":"709.538321ms","remote":"127.0.0.1:48522","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-31T00:40:20.669491Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:19.877281Z","time spent":"792.179527ms","remote":"127.0.0.1:49372","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":693,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-31T00:40:20.671641Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":5706233631807294029,"received-request-id":5706233631807294026} {"level":"warn","ts":"2026-03-31T00:40:20.673300Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"248.893577ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:436"} {"level":"info","ts":"2026-03-31T00:40:20.673363Z","caller":"traceutil/trace.go:172","msg":"trace[1708159554] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:596; }","duration":"248.96838ms","start":"2026-03-31T00:40:20.424380Z","end":"2026-03-31T00:40:20.673349Z","steps":["trace[1708159554] 'agreement among raft nodes before linearized reading' (duration: 248.766844ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:21.794493Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"174.610984ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:21.794587Z","caller":"traceutil/trace.go:172","msg":"trace[2013603066] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:597; }","duration":"174.726707ms","start":"2026-03-31T00:40:21.619842Z","end":"2026-03-31T00:40:21.794568Z","steps":["trace[2013603066] 'range keys from in-memory index tree' (duration: 174.537232ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:21.794675Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"141.946616ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-31T00:40:21.794813Z","caller":"traceutil/trace.go:172","msg":"trace[1521998927] linearizableReadLoop","detail":"{readStateIndex:639; appliedIndex:638; }","duration":"358.476958ms","start":"2026-03-31T00:40:21.436316Z","end":"2026-03-31T00:40:21.794793Z","steps":["trace[1521998927] 'read index received' (duration: 217.516032ms)","trace[1521998927] 'applied index is now lower than readState.Index' (duration: 140.959636ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:21.794979Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"358.638083ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceslices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:21.795038Z","caller":"traceutil/trace.go:172","msg":"trace[473557784] range","detail":"{range_begin:/registry/resourceslices; range_end:; response_count:0; response_revision:598; }","duration":"358.711265ms","start":"2026-03-31T00:40:21.436309Z","end":"2026-03-31T00:40:21.795021Z","steps":["trace[473557784] 'agreement among raft nodes before linearized reading' (duration: 358.585571ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:21.795083Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:21.436282Z","time spent":"358.791168ms","remote":"127.0.0.1:53364","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/resourceslices\" limit:1 "} {"level":"info","ts":"2026-03-31T00:40:22.589369Z","caller":"traceutil/trace.go:172","msg":"trace[1818987111] linearizableReadLoop","detail":"{readStateIndex:641; appliedIndex:641; }","duration":"381.019663ms","start":"2026-03-31T00:40:22.208309Z","end":"2026-03-31T00:40:22.589328Z","steps":["trace[1818987111] 'read index received' (duration: 380.996722ms)","trace[1818987111] 'applied index is now lower than readState.Index' (duration: 20.711µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:22.589761Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"381.376264ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-31T00:40:22.589997Z","caller":"traceutil/trace.go:172","msg":"trace[1947894007] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:600; }","duration":"381.671342ms","start":"2026-03-31T00:40:22.208304Z","end":"2026-03-31T00:40:22.589975Z","steps":["trace[1947894007] 'agreement among raft nodes before linearized reading' (duration: 381.157447ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:22.590117Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:22.208285Z","time spent":"381.815416ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:40:22.591765Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"348.320873ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:508"} {"level":"info","ts":"2026-03-31T00:40:22.591820Z","caller":"traceutil/trace.go:172","msg":"trace[1428977141] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:601; }","duration":"348.378235ms","start":"2026-03-31T00:40:22.243430Z","end":"2026-03-31T00:40:22.591809Z","steps":["trace[1428977141] 'agreement among raft nodes before linearized reading' (duration: 348.287052ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:22.591849Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:22.243412Z","time spent":"348.429926ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":531,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:40:23.988473Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"157.64071ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-31T00:40:24.328312Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"117.038889ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-31T00:40:24.328577Z","caller":"traceutil/trace.go:172","msg":"trace[120986344] linearizableReadLoop","detail":"{readStateIndex:645; appliedIndex:644; }","duration":"536.228432ms","start":"2026-03-31T00:40:23.792305Z","end":"2026-03-31T00:40:24.328533Z","steps":["trace[120986344] 'read index received' (duration: 459.778828ms)","trace[120986344] 'applied index is now lower than readState.Index' (duration: 76.446224ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:24.328843Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"536.484048ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/limitranges\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:24.329237Z","caller":"traceutil/trace.go:172","msg":"trace[907306096] range","detail":"{range_begin:/registry/limitranges; range_end:; response_count:0; response_revision:605; }","duration":"536.923102ms","start":"2026-03-31T00:40:23.792289Z","end":"2026-03-31T00:40:24.329212Z","steps":["trace[907306096] 'agreement among raft nodes before linearized reading' (duration: 536.405226ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:24.329318Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:23.792256Z","time spent":"537.047335ms","remote":"127.0.0.1:48846","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":28,"request content":"key:\"/registry/limitranges\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:40:24.331769Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"240.379765ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts/kube-system/kube-proxy\" limit:1 ","response":"range_response_count:1 size:185"} {"level":"info","ts":"2026-03-31T00:40:24.331823Z","caller":"traceutil/trace.go:172","msg":"trace[551892868] range","detail":"{range_begin:/registry/serviceaccounts/kube-system/kube-proxy; range_end:; response_count:1; response_revision:606; }","duration":"240.439836ms","start":"2026-03-31T00:40:24.091366Z","end":"2026-03-31T00:40:24.331806Z","steps":["trace[551892868] 'agreement among raft nodes before linearized reading' (duration: 240.294232ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:24.332162Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"369.989595ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:24.332289Z","caller":"traceutil/trace.go:172","msg":"trace[1244208323] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:606; }","duration":"370.082578ms","start":"2026-03-31T00:40:23.962160Z","end":"2026-03-31T00:40:24.332242Z","steps":["trace[1244208323] 'agreement among raft nodes before linearized reading' (duration: 369.429548ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:24.332324Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:23.962120Z","time spent":"370.193811ms","remote":"127.0.0.1:48522","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-31T00:40:24.469439Z","caller":"traceutil/trace.go:172","msg":"trace[130726782] linearizableReadLoop","detail":"{readStateIndex:650; appliedIndex:650; }","duration":"135.842784ms","start":"2026-03-31T00:40:24.333569Z","end":"2026-03-31T00:40:24.469412Z","steps":["trace[130726782] 'read index received' (duration: 135.835984ms)","trace[130726782] 'applied index is now lower than readState.Index' (duration: 5.46µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:24.516794Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"183.193949ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:24.516889Z","caller":"traceutil/trace.go:172","msg":"trace[1148841806] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:607; }","duration":"183.311872ms","start":"2026-03-31T00:40:24.333557Z","end":"2026-03-31T00:40:24.516868Z","steps":["trace[1148841806] 'agreement among raft nodes before linearized reading' (duration: 135.993068ms)","trace[1148841806] 'range keys from in-memory index tree' (duration: 47.16135ms)"],"step_count":2} {"level":"info","ts":"2026-03-31T00:40:24.648624Z","caller":"traceutil/trace.go:172","msg":"trace[154550118] transaction","detail":"{read_only:false; response_revision:608; number_of_response:1; }","duration":"313.575077ms","start":"2026-03-31T00:40:24.335028Z","end":"2026-03-31T00:40:24.648603Z","steps":["trace[154550118] 'process raft request' (duration: 312.937438ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:24.648758Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:24.335016Z","time spent":"313.68514ms","remote":"127.0.0.1:48942","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":5683,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-31T00:40:24.855602Z","caller":"traceutil/trace.go:172","msg":"trace[1385993007] linearizableReadLoop","detail":"{readStateIndex:652; appliedIndex:651; }","duration":"386.033797ms","start":"2026-03-31T00:40:24.469525Z","end":"2026-03-31T00:40:24.855559Z","steps":["trace[1385993007] 'read index received' (duration: 340.780101ms)","trace[1385993007] 'applied index is now lower than readState.Index' (duration: 45.252536ms)"],"step_count":2} {"level":"info","ts":"2026-03-31T00:40:24.855758Z","caller":"traceutil/trace.go:172","msg":"trace[1415869952] transaction","detail":"{read_only:false; response_revision:609; number_of_response:1; }","duration":"502.319155ms","start":"2026-03-31T00:40:24.353415Z","end":"2026-03-31T00:40:24.855734Z","steps":["trace[1415869952] 'process raft request' (duration: 436.553225ms)","trace[1415869952] 'compare' (duration: 65.438631ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:24.855796Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"521.928992ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-proxy-qpl7r\" limit:1 ","response":"range_response_count:1 size:5044"} {"level":"info","ts":"2026-03-31T00:40:24.855831Z","caller":"traceutil/trace.go:172","msg":"trace[1608107710] range","detail":"{range_begin:/registry/pods/kube-system/kube-proxy-qpl7r; range_end:; response_count:1; response_revision:609; }","duration":"521.983994ms","start":"2026-03-31T00:40:24.333837Z","end":"2026-03-31T00:40:24.855821Z","steps":["trace[1608107710] 'agreement among raft nodes before linearized reading' (duration: 521.887442ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:24.855865Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:24.333826Z","time spent":"522.029225ms","remote":"127.0.0.1:48942","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":5067,"request content":"key:\"/registry/pods/kube-system/kube-proxy-qpl7r\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:40:24.855882Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:24.353369Z","time spent":"502.439298ms","remote":"127.0.0.1:48942","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":5029,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-31T00:40:24.856044Z","caller":"traceutil/trace.go:172","msg":"trace[260227839] transaction","detail":"{read_only:false; response_revision:611; number_of_response:1; }","duration":"200.740629ms","start":"2026-03-31T00:40:24.655273Z","end":"2026-03-31T00:40:24.856014Z","steps":["trace[260227839] 'process raft request' (duration: 200.635996ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:24.857916Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"237.736831ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:24.858033Z","caller":"traceutil/trace.go:172","msg":"trace[805030319] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:611; }","duration":"237.873335ms","start":"2026-03-31T00:40:24.620137Z","end":"2026-03-31T00:40:24.858010Z","steps":["trace[805030319] 'agreement among raft nodes before linearized reading' (duration: 237.68839ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:40:25.058227Z","caller":"traceutil/trace.go:172","msg":"trace[546011926] transaction","detail":"{read_only:false; response_revision:613; number_of_response:1; }","duration":"169.407344ms","start":"2026-03-31T00:40:24.888793Z","end":"2026-03-31T00:40:25.058200Z","steps":["trace[546011926] 'process raft request' (duration: 119.92077ms)","trace[546011926] 'compare' (duration: 49.101473ms)"],"step_count":2} {"level":"info","ts":"2026-03-31T00:40:25.266999Z","caller":"traceutil/trace.go:172","msg":"trace[2047466584] linearizableReadLoop","detail":"{readStateIndex:657; appliedIndex:656; }","duration":"109.635596ms","start":"2026-03-31T00:40:25.157337Z","end":"2026-03-31T00:40:25.266972Z","steps":["trace[2047466584] 'read index received' (duration: 87.013519ms)","trace[2047466584] 'applied index is now lower than readState.Index' (duration: 22.620947ms)"],"step_count":2} {"level":"info","ts":"2026-03-31T00:40:25.267070Z","caller":"traceutil/trace.go:172","msg":"trace[1504700195] transaction","detail":"{read_only:false; response_revision:614; number_of_response:1; }","duration":"200.011109ms","start":"2026-03-31T00:40:25.067020Z","end":"2026-03-31T00:40:25.267031Z","steps":["trace[1504700195] 'process raft request' (duration: 176.400025ms)","trace[1504700195] 'compare' (duration: 23.265775ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:25.267251Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"109.887042ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-31T00:40:25.267296Z","caller":"traceutil/trace.go:172","msg":"trace[2085413345] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:614; }","duration":"109.966514ms","start":"2026-03-31T00:40:25.157318Z","end":"2026-03-31T00:40:25.267284Z","steps":["trace[2085413345] 'agreement among raft nodes before linearized reading' (duration: 109.776869ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:25.601244Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"312.735213ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"warn","ts":"2026-03-31T00:40:26.224863Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"264.913814ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:26.224961Z","caller":"traceutil/trace.go:172","msg":"trace[1768720355] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:615; }","duration":"265.018567ms","start":"2026-03-31T00:40:25.959920Z","end":"2026-03-31T00:40:26.224939Z","steps":["trace[1768720355] 'range keys from in-memory index tree' (duration: 262.755564ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:40:26.736979Z","caller":"traceutil/trace.go:172","msg":"trace[832715119] transaction","detail":"{read_only:false; response_revision:618; number_of_response:1; }","duration":"203.437639ms","start":"2026-03-31T00:40:26.533515Z","end":"2026-03-31T00:40:26.736953Z","steps":["trace[832715119] 'process raft request' (duration: 193.512471ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:40:26.736983Z","caller":"traceutil/trace.go:172","msg":"trace[815843826] linearizableReadLoop","detail":"{readStateIndex:662; appliedIndex:661; }","duration":"117.346151ms","start":"2026-03-31T00:40:26.619610Z","end":"2026-03-31T00:40:26.736956Z","steps":["trace[815843826] 'read index received' (duration: 108.090553ms)","trace[815843826] 'applied index is now lower than readState.Index' (duration: 9.236667ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:26.737358Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"117.726641ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:26.737770Z","caller":"traceutil/trace.go:172","msg":"trace[699609181] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:618; }","duration":"118.155373ms","start":"2026-03-31T00:40:26.619594Z","end":"2026-03-31T00:40:26.737749Z","steps":["trace[699609181] 'agreement among raft nodes before linearized reading' (duration: 117.68319ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:28.423430Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294070,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:40:28.923932Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294070,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:40:28.967850Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.095576658s","expected-duration":"1s"} {"level":"info","ts":"2026-03-31T00:40:29.126541Z","caller":"traceutil/trace.go:172","msg":"trace[892463991] linearizableReadLoop","detail":"{readStateIndex:669; appliedIndex:670; }","duration":"1.203845407s","start":"2026-03-31T00:40:27.922673Z","end":"2026-03-31T00:40:29.126519Z","steps":["trace[892463991] 'read index received' (duration: 1.203827386s)","trace[892463991] 'applied index is now lower than readState.Index' (duration: 16.691µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:29.127317Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.204626037s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/etcd-controller-2\" limit:1 ","response":"range_response_count:1 size:4429"} {"level":"info","ts":"2026-03-31T00:40:29.127368Z","caller":"traceutil/trace.go:172","msg":"trace[1544527819] range","detail":"{range_begin:/registry/pods/kube-system/etcd-controller-2; range_end:; response_count:1; response_revision:625; }","duration":"1.204694099s","start":"2026-03-31T00:40:27.922662Z","end":"2026-03-31T00:40:29.127356Z","steps":["trace[1544527819] 'agreement among raft nodes before linearized reading' (duration: 1.203958029s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:29.127405Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:27.922415Z","time spent":"1.204981407s","remote":"127.0.0.1:48942","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":4452,"request content":"key:\"/registry/pods/kube-system/etcd-controller-2\" limit:1 "} {"level":"info","ts":"2026-03-31T00:40:29.127901Z","caller":"traceutil/trace.go:172","msg":"trace[1870321667] transaction","detail":"{read_only:false; response_revision:627; number_of_response:1; }","duration":"217.834052ms","start":"2026-03-31T00:40:28.910054Z","end":"2026-03-31T00:40:29.127888Z","steps":["trace[1870321667] 'process raft request' (duration: 217.628717ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:29.128313Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.170074494s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-31T00:40:29.128323Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"159.11681ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-31T00:40:29.128362Z","caller":"traceutil/trace.go:172","msg":"trace[1108073240] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:627; }","duration":"1.170194188s","start":"2026-03-31T00:40:27.958155Z","end":"2026-03-31T00:40:29.128349Z","steps":["trace[1108073240] 'agreement among raft nodes before linearized reading' (duration: 1.170046144s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:29.128398Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:27.958132Z","time spent":"1.170257779s","remote":"127.0.0.1:48548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-31T00:40:29.128377Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"508.789758ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:29.128397Z","caller":"traceutil/trace.go:172","msg":"trace[136604677] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:627; }","duration":"159.204332ms","start":"2026-03-31T00:40:28.969176Z","end":"2026-03-31T00:40:29.128380Z","steps":["trace[136604677] 'agreement among raft nodes before linearized reading' (duration: 159.075569ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:40:29.128450Z","caller":"traceutil/trace.go:172","msg":"trace[1365480614] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:627; }","duration":"508.87327ms","start":"2026-03-31T00:40:28.619561Z","end":"2026-03-31T00:40:29.128434Z","steps":["trace[1365480614] 'agreement among raft nodes before linearized reading' (duration: 508.746226ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:29.128754Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"136.76886ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-31T00:40:29.128824Z","caller":"traceutil/trace.go:172","msg":"trace[1592757218] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:627; }","duration":"136.848293ms","start":"2026-03-31T00:40:28.991958Z","end":"2026-03-31T00:40:29.128807Z","steps":["trace[1592757218] 'agreement among raft nodes before linearized reading' (duration: 136.297168ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:34.668811Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"102.384163ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:481"} {"level":"info","ts":"2026-03-31T00:40:34.668873Z","caller":"traceutil/trace.go:172","msg":"trace[1422917563] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:648; }","duration":"102.468925ms","start":"2026-03-31T00:40:34.566390Z","end":"2026-03-31T00:40:34.668859Z","steps":["trace[1422917563] 'range keys from in-memory index tree' (duration: 100.605878ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:40:39.870619Z","caller":"traceutil/trace.go:172","msg":"trace[1833743858] transaction","detail":"{read_only:false; response_revision:661; number_of_response:1; }","duration":"562.205193ms","start":"2026-03-31T00:40:39.308392Z","end":"2026-03-31T00:40:39.870597Z","steps":["trace[1833743858] 'process raft request' (duration: 562.117191ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:39.870778Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:39.308358Z","time spent":"562.335506ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-31T00:40:39.871202Z","caller":"traceutil/trace.go:172","msg":"trace[1343023066] linearizableReadLoop","detail":"{readStateIndex:711; appliedIndex:711; }","duration":"209.567458ms","start":"2026-03-31T00:40:39.661611Z","end":"2026-03-31T00:40:39.871178Z","steps":["trace[1343023066] 'read index received' (duration: 209.561428ms)","trace[1343023066] 'applied index is now lower than readState.Index' (duration: 4.94µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:39.871484Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"209.862546ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:39.871574Z","caller":"traceutil/trace.go:172","msg":"trace[1267763122] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:661; }","duration":"209.954988ms","start":"2026-03-31T00:40:39.661604Z","end":"2026-03-31T00:40:39.871559Z","steps":["trace[1267763122] 'agreement among raft nodes before linearized reading' (duration: 209.749053ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:40.218088Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"112.40217ms","expected-duration":"100ms","prefix":"","request":"header: lease_grant:","response":"size:40"} {"level":"warn","ts":"2026-03-31T00:40:40.218240Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:40:39.873855Z","time spent":"344.378014ms","remote":"127.0.0.1:48618","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-31T00:40:40.218431Z","caller":"traceutil/trace.go:172","msg":"trace[909754680] linearizableReadLoop","detail":"{readStateIndex:712; appliedIndex:711; }","duration":"258.771635ms","start":"2026-03-31T00:40:39.959633Z","end":"2026-03-31T00:40:40.218405Z","steps":["trace[909754680] 'read index received' (duration: 147.314149ms)","trace[909754680] 'applied index is now lower than readState.Index' (duration: 111.455166ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:40:40.218695Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"259.020681ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:40:40.218739Z","caller":"traceutil/trace.go:172","msg":"trace[813995863] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:662; }","duration":"259.107084ms","start":"2026-03-31T00:40:39.959621Z","end":"2026-03-31T00:40:40.218728Z","steps":["trace[813995863] 'agreement among raft nodes before linearized reading' (duration: 258.898918ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:40:53.243026Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.235:40106","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-31T00:40:53.673002Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"3119206cae914f30 switched to configuration voters=(3537894633433550640 11609189331680412151) learners=(4625522210060070399)"} {"level":"info","ts":"2026-03-31T00:40:53.673123Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"97da89d0cbebd2b5","local-member-id":"3119206cae914f30","added-peer-id":"403127f1674f6dff","added-peer-peer-urls":["https://199.204.45.235:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-03-31T00:40:53.673149Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:53.673166Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:53.673728Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:53.678782Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:53.678864Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff","remote-peer-urls":["https://199.204.45.235:2380"]} {"level":"info","ts":"2026-03-31T00:40:53.679029Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:53.679135Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"3119206cae914f30","raft-conf-change":"ConfChangeAddLearnerNode","raft-conf-change-node-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:53.679152Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:53.679281Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"warn","ts":"2026-03-31T00:40:53.690697Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.235:40180","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:53.707333Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.235:40228","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:54.218235Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.235:40232","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:54.708361Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.235:40254","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:55.720200Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.235:40298","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-31T00:40:56.258590Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:56.258686Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:56.258735Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:56.264328Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"3119206cae914f30","to":"403127f1674f6dff","stream-type":"stream Message"} {"level":"info","ts":"2026-03-31T00:40:56.264376Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:56.265132Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"3119206cae914f30","to":"403127f1674f6dff","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-31T00:40:56.265181Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"warn","ts":"2026-03-31T00:40:56.296160Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:40:56.296162Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff","error":"EOF"} {"level":"info","ts":"2026-03-31T00:40:56.307534Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"3119206cae914f30","to":"403127f1674f6dff","stream-type":"stream Message"} {"level":"warn","ts":"2026-03-31T00:40:56.307555Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:56.307564Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:56.310428Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:56.312007Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"3119206cae914f30","to":"403127f1674f6dff","stream-type":"stream MsgApp v2"} {"level":"warn","ts":"2026-03-31T00:40:56.312032Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:56.312042Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:56.318160Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"3119206cae914f30","remote-peer-id":"403127f1674f6dff"} {"level":"info","ts":"2026-03-31T00:40:56.760059Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"3119206cae914f30 switched to configuration voters=(3537894633433550640 4625522210060070399 11609189331680412151)"} {"level":"info","ts":"2026-03-31T00:40:56.760250Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"97da89d0cbebd2b5","local-member-id":"3119206cae914f30","promoted-member-id":"403127f1674f6dff"} {"level":"warn","ts":"2026-03-31T00:40:56.787504Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.235:40350","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:41:01.094733Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"135.232892ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:01.094828Z","caller":"traceutil/trace.go:172","msg":"trace[888640056] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:723; }","duration":"135.340244ms","start":"2026-03-31T00:41:00.959470Z","end":"2026-03-31T00:41:01.094811Z","steps":["trace[888640056] 'range keys from in-memory index tree' (duration: 133.424894ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:08.985566Z","caller":"traceutil/trace.go:172","msg":"trace[580767887] linearizableReadLoop","detail":"{readStateIndex:882; appliedIndex:881; }","duration":"207.54006ms","start":"2026-03-31T00:41:08.778003Z","end":"2026-03-31T00:41:08.985543Z","steps":["trace[580767887] 'read index received' (duration: 188.111808ms)","trace[580767887] 'applied index is now lower than readState.Index' (duration: 19.426922ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:08.985715Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"207.685542ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/controllerrevisions\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:08.985802Z","caller":"traceutil/trace.go:172","msg":"trace[1161288164] range","detail":"{range_begin:/registry/controllerrevisions; range_end:; response_count:0; response_revision:815; }","duration":"207.785804ms","start":"2026-03-31T00:41:08.777996Z","end":"2026-03-31T00:41:08.985782Z","steps":["trace[1161288164] 'agreement among raft nodes before linearized reading' (duration: 207.632471ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:08.987812Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"115.05834ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/statefulsets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:08.987859Z","caller":"traceutil/trace.go:172","msg":"trace[808813711] range","detail":"{range_begin:/registry/statefulsets; range_end:; response_count:0; response_revision:815; }","duration":"115.109031ms","start":"2026-03-31T00:41:08.872739Z","end":"2026-03-31T00:41:08.987848Z","steps":["trace[808813711] 'agreement among raft nodes before linearized reading' (duration: 115.03392ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:09.795067Z","caller":"traceutil/trace.go:172","msg":"trace[305667147] linearizableReadLoop","detail":"{readStateIndex:883; appliedIndex:885; }","duration":"175.387328ms","start":"2026-03-31T00:41:09.619654Z","end":"2026-03-31T00:41:09.795041Z","steps":["trace[305667147] 'read index received' (duration: 174.997561ms)","trace[305667147] 'applied index is now lower than readState.Index' (duration: 388.207µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:09.795222Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"175.52399ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:09.795341Z","caller":"traceutil/trace.go:172","msg":"trace[631393697] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:818; }","duration":"175.680313ms","start":"2026-03-31T00:41:09.619641Z","end":"2026-03-31T00:41:09.795321Z","steps":["trace[631393697] 'agreement among raft nodes before linearized reading' (duration: 175.49592ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:10.170555Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:09.801161Z","time spent":"369.390297ms","remote":"127.0.0.1:48618","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-31T00:41:10.170590Z","caller":"traceutil/trace.go:172","msg":"trace[1337304300] linearizableReadLoop","detail":"{readStateIndex:889; appliedIndex:888; }","duration":"315.560535ms","start":"2026-03-31T00:41:09.855004Z","end":"2026-03-31T00:41:10.170564Z","steps":["trace[1337304300] 'read index received' (duration: 310.922878ms)","trace[1337304300] 'applied index is now lower than readState.Index' (duration: 4.635277ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:10.170859Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"315.79657ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/statefulsets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:10.170989Z","caller":"traceutil/trace.go:172","msg":"trace[1613637393] range","detail":"{range_begin:/registry/statefulsets; range_end:; response_count:0; response_revision:821; }","duration":"315.983055ms","start":"2026-03-31T00:41:09.854988Z","end":"2026-03-31T00:41:10.170971Z","steps":["trace[1613637393] 'agreement among raft nodes before linearized reading' (duration: 315.706788ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:10.171068Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:09.854965Z","time spent":"316.090826ms","remote":"127.0.0.1:49442","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":28,"request content":"key:\"/registry/statefulsets\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:10.172645Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"107.78665ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"warn","ts":"2026-03-31T00:41:10.172634Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"211.735425ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:10.172686Z","caller":"traceutil/trace.go:172","msg":"trace[2009149267] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:821; }","duration":"107.837291ms","start":"2026-03-31T00:41:10.064839Z","end":"2026-03-31T00:41:10.172676Z","steps":["trace[2009149267] 'agreement among raft nodes before linearized reading' (duration: 107.720819ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:10.172705Z","caller":"traceutil/trace.go:172","msg":"trace[1504739364] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:821; }","duration":"211.823726ms","start":"2026-03-31T00:41:09.960863Z","end":"2026-03-31T00:41:10.172687Z","steps":["trace[1504739364] 'agreement among raft nodes before linearized reading' (duration: 211.681864ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:22.152244Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"103.632221ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-31T00:41:28.006177Z","caller":"traceutil/trace.go:172","msg":"trace[1045019221] linearizableReadLoop","detail":"{readStateIndex:986; appliedIndex:986; }","duration":"102.082906ms","start":"2026-03-31T00:41:27.904041Z","end":"2026-03-31T00:41:28.006124Z","steps":["trace[1045019221] 'read index received' (duration: 102.076516ms)","trace[1045019221] 'applied index is now lower than readState.Index' (duration: 5.23µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:28.006343Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"102.288319ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-31T00:41:28.006376Z","caller":"traceutil/trace.go:172","msg":"trace[648046957] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:906; }","duration":"102.34011ms","start":"2026-03-31T00:41:27.904029Z","end":"2026-03-31T00:41:28.006369Z","steps":["trace[648046957] 'agreement among raft nodes before linearized reading' (duration: 102.243818ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:28.092900Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"135.395794ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:28.092955Z","caller":"traceutil/trace.go:172","msg":"trace[384998279] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:908; }","duration":"135.460395ms","start":"2026-03-31T00:41:27.957484Z","end":"2026-03-31T00:41:28.092945Z","steps":["trace[384998279] 'agreement among raft nodes before linearized reading' (duration: 135.354443ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:37.102385Z","caller":"traceutil/trace.go:172","msg":"trace[1621064958] transaction","detail":"{read_only:false; response_revision:976; number_of_response:1; }","duration":"307.545171ms","start":"2026-03-31T00:41:36.794819Z","end":"2026-03-31T00:41:37.102365Z","steps":["trace[1621064958] 'process raft request' (duration: 307.409669ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:37.102522Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:36.794796Z","time spent":"307.661153ms","remote":"127.0.0.1:48942","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":5187,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-31T00:41:37.188203Z","caller":"traceutil/trace.go:172","msg":"trace[533325602] linearizableReadLoop","detail":"{readStateIndex:1062; appliedIndex:1062; }","duration":"276.115577ms","start":"2026-03-31T00:41:36.912072Z","end":"2026-03-31T00:41:37.188188Z","steps":["trace[533325602] 'read index received' (duration: 276.111657ms)","trace[533325602] 'applied index is now lower than readState.Index' (duration: 3.11µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:37.188515Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"276.416002ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts/kube-system/cilium-operator\" limit:1 ","response":"range_response_count:1 size:538"} {"level":"info","ts":"2026-03-31T00:41:37.188593Z","caller":"traceutil/trace.go:172","msg":"trace[1136022977] range","detail":"{range_begin:/registry/serviceaccounts/kube-system/cilium-operator; range_end:; response_count:1; response_revision:977; }","duration":"276.515553ms","start":"2026-03-31T00:41:36.912061Z","end":"2026-03-31T00:41:37.188577Z","steps":["trace[1136022977] 'agreement among raft nodes before linearized reading' (duration: 276.263999ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:37.189934Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"231.319468ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:37.190006Z","caller":"traceutil/trace.go:172","msg":"trace[1171368221] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:977; }","duration":"231.40571ms","start":"2026-03-31T00:41:36.958585Z","end":"2026-03-31T00:41:37.189991Z","steps":["trace[1171368221] 'agreement among raft nodes before linearized reading' (duration: 231.281817ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:37.190057Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"161.62172ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:436"} {"level":"info","ts":"2026-03-31T00:41:37.190107Z","caller":"traceutil/trace.go:172","msg":"trace[1004046715] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:977; }","duration":"161.67591ms","start":"2026-03-31T00:41:37.028419Z","end":"2026-03-31T00:41:37.190095Z","steps":["trace[1004046715] 'agreement among raft nodes before linearized reading' (duration: 161.500497ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:37.190254Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"175.725621ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts/kube-system/cilium\" limit:1 ","response":"range_response_count:1 size:520"} {"level":"info","ts":"2026-03-31T00:41:37.190349Z","caller":"traceutil/trace.go:172","msg":"trace[1261660693] range","detail":"{range_begin:/registry/serviceaccounts/kube-system/cilium; range_end:; response_count:1; response_revision:977; }","duration":"175.782361ms","start":"2026-03-31T00:41:37.014510Z","end":"2026-03-31T00:41:37.190293Z","steps":["trace[1261660693] 'agreement among raft nodes before linearized reading' (duration: 175.560068ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:38.703808Z","caller":"traceutil/trace.go:172","msg":"trace[564430996] linearizableReadLoop","detail":"{readStateIndex:1073; appliedIndex:1073; }","duration":"141.801145ms","start":"2026-03-31T00:41:38.561987Z","end":"2026-03-31T00:41:38.703788Z","steps":["trace[564430996] 'read index received' (duration: 141.796185ms)","trace[564430996] 'applied index is now lower than readState.Index' (duration: 3.88µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:38.704067Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"142.06351ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:481"} {"level":"info","ts":"2026-03-31T00:41:38.704155Z","caller":"traceutil/trace.go:172","msg":"trace[1326586674] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:986; }","duration":"142.166921ms","start":"2026-03-31T00:41:38.561973Z","end":"2026-03-31T00:41:38.704140Z","steps":["trace[1326586674] 'agreement among raft nodes before linearized reading' (duration: 141.924998ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:39.080545Z","caller":"traceutil/trace.go:172","msg":"trace[1755499428] linearizableReadLoop","detail":"{readStateIndex:1075; appliedIndex:1074; }","duration":"116.585653ms","start":"2026-03-31T00:41:38.963934Z","end":"2026-03-31T00:41:39.080519Z","steps":["trace[1755499428] 'read index received' (duration: 115.089891ms)","trace[1755499428] 'applied index is now lower than readState.Index' (duration: 1.492982ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:39.082422Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"118.476853ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:39.082507Z","caller":"traceutil/trace.go:172","msg":"trace[1768940719] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:988; }","duration":"118.565424ms","start":"2026-03-31T00:41:38.963922Z","end":"2026-03-31T00:41:39.082488Z","steps":["trace[1768940719] 'agreement among raft nodes before linearized reading' (duration: 116.751026ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:40.221278Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294494,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:40.721884Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294494,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:41.216388Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.553735407s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-31T00:41:41.222385Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294494,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-31T00:41:41.322135Z","caller":"traceutil/trace.go:172","msg":"trace[369865661] linearizableReadLoop","detail":"{readStateIndex:1090; appliedIndex:1093; }","duration":"1.601655752s","start":"2026-03-31T00:41:39.720392Z","end":"2026-03-31T00:41:41.322048Z","steps":["trace[369865661] 'read index received' (duration: 1.601640492s)","trace[369865661] 'applied index is now lower than readState.Index' (duration: 12.83µs)"],"step_count":2} {"level":"info","ts":"2026-03-31T00:41:41.322755Z","caller":"traceutil/trace.go:172","msg":"trace[480073481] transaction","detail":"{read_only:false; response_revision:1008; number_of_response:1; }","duration":"316.049711ms","start":"2026-03-31T00:41:41.006691Z","end":"2026-03-31T00:41:41.322741Z","steps":["trace[480073481] 'process raft request' (duration: 315.775787ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:41.322883Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:41.006665Z","time spent":"316.152622ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":536,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-31T00:41:41.323709Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.603306967s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:508"} {"level":"info","ts":"2026-03-31T00:41:41.323757Z","caller":"traceutil/trace.go:172","msg":"trace[1386687596] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1008; }","duration":"1.603356048s","start":"2026-03-31T00:41:39.720388Z","end":"2026-03-31T00:41:41.323744Z","steps":["trace[1386687596] 'agreement among raft nodes before linearized reading' (duration: 1.602100469s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:41.323788Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:39.720369Z","time spent":"1.603409608s","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":531,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"info","ts":"2026-03-31T00:41:41.333972Z","caller":"traceutil/trace.go:172","msg":"trace[1572145845] transaction","detail":"{read_only:false; number_of_response:0; response_revision:1016; }","duration":"111.433671ms","start":"2026-03-31T00:41:41.222513Z","end":"2026-03-31T00:41:41.333947Z","steps":["trace[1572145845] 'process raft request' (duration: 111.321689ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:41.334043Z","caller":"traceutil/trace.go:172","msg":"trace[1779822233] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1016; }","duration":"108.414794ms","start":"2026-03-31T00:41:41.225602Z","end":"2026-03-31T00:41:41.334017Z","steps":["trace[1779822233] 'process raft request' (duration: 108.288902ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:41.333998Z","caller":"traceutil/trace.go:172","msg":"trace[84152202] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1016; }","duration":"108.417725ms","start":"2026-03-31T00:41:41.225566Z","end":"2026-03-31T00:41:41.333984Z","steps":["trace[84152202] 'process raft request' (duration: 108.295823ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:41.345751Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"364.08335ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:436"} {"level":"warn","ts":"2026-03-31T00:41:41.345752Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"381.221474ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-31T00:41:41.345760Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"726.388542ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:41.345809Z","caller":"traceutil/trace.go:172","msg":"trace[302585081] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1018; }","duration":"364.144071ms","start":"2026-03-31T00:41:40.981654Z","end":"2026-03-31T00:41:41.345798Z","steps":["trace[302585081] 'agreement among raft nodes before linearized reading' (duration: 364.056759ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:41.345833Z","caller":"traceutil/trace.go:172","msg":"trace[974103186] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1018; }","duration":"726.470433ms","start":"2026-03-31T00:41:40.619351Z","end":"2026-03-31T00:41:41.345821Z","steps":["trace[974103186] 'agreement among raft nodes before linearized reading' (duration: 726.362521ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:41.345839Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:40.981633Z","time spent":"364.196821ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":459,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:41.345984Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.553135544s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.116\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-31T00:41:41.346018Z","caller":"traceutil/trace.go:172","msg":"trace[1266011874] range","detail":"{range_begin:/registry/masterleases/199.204.45.116; range_end:; response_count:1; response_revision:1018; }","duration":"1.553171095s","start":"2026-03-31T00:41:39.792839Z","end":"2026-03-31T00:41:41.346010Z","steps":["trace[1266011874] 'agreement among raft nodes before linearized reading' (duration: 1.553079954s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:41.346034Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:39.792817Z","time spent":"1.553211805s","remote":"127.0.0.1:48618","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.116\" limit:1 "} {"level":"info","ts":"2026-03-31T00:41:41.345816Z","caller":"traceutil/trace.go:172","msg":"trace[439948465] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1018; }","duration":"381.292456ms","start":"2026-03-31T00:41:40.964512Z","end":"2026-03-31T00:41:41.345804Z","steps":["trace[439948465] 'agreement among raft nodes before linearized reading' (duration: 381.185654ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:41.346161Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:40.964491Z","time spent":"381.659951ms","remote":"127.0.0.1:48522","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-31T00:41:43.686139Z","caller":"traceutil/trace.go:172","msg":"trace[1669698141] linearizableReadLoop","detail":"{readStateIndex:1241; appliedIndex:1240; }","duration":"110.665289ms","start":"2026-03-31T00:41:43.575456Z","end":"2026-03-31T00:41:43.686121Z","steps":["trace[1669698141] 'read index received' (duration: 109.989199ms)","trace[1669698141] 'applied index is now lower than readState.Index' (duration: 675.34µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:43.686387Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"110.908643ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:508"} {"level":"info","ts":"2026-03-31T00:41:43.686427Z","caller":"traceutil/trace.go:172","msg":"trace[1914378160] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1076; }","duration":"110.964674ms","start":"2026-03-31T00:41:43.575451Z","end":"2026-03-31T00:41:43.686416Z","steps":["trace[1914378160] 'agreement among raft nodes before linearized reading' (duration: 110.78896ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:45.782262Z","caller":"traceutil/trace.go:172","msg":"trace[271444638] linearizableReadLoop","detail":"{readStateIndex:1258; appliedIndex:1258; }","duration":"163.135589ms","start":"2026-03-31T00:41:45.619070Z","end":"2026-03-31T00:41:45.782206Z","steps":["trace[271444638] 'read index received' (duration: 163.129309ms)","trace[271444638] 'applied index is now lower than readState.Index' (duration: 5.22µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:45.782385Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"163.299372ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:45.782464Z","caller":"traceutil/trace.go:172","msg":"trace[621411035] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1090; }","duration":"163.384743ms","start":"2026-03-31T00:41:45.619063Z","end":"2026-03-31T00:41:45.782448Z","steps":["trace[621411035] 'agreement among raft nodes before linearized reading' (duration: 163.271641ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:46.402270Z","caller":"traceutil/trace.go:172","msg":"trace[1501476515] linearizableReadLoop","detail":"{readStateIndex:1261; appliedIndex:1261; }","duration":"468.189356ms","start":"2026-03-31T00:41:45.934003Z","end":"2026-03-31T00:41:46.402192Z","steps":["trace[1501476515] 'read index received' (duration: 468.177736ms)","trace[1501476515] 'applied index is now lower than readState.Index' (duration: 9.42µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:46.431536Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"497.506013ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"warn","ts":"2026-03-31T00:41:46.431709Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"471.816968ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:46.431776Z","caller":"traceutil/trace.go:172","msg":"trace[322547155] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1093; }","duration":"497.619044ms","start":"2026-03-31T00:41:45.933998Z","end":"2026-03-31T00:41:46.431617Z","steps":["trace[322547155] 'agreement among raft nodes before linearized reading' (duration: 468.335948ms)","trace[322547155] 'range keys from in-memory index tree' (duration: 29.124474ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:46.431860Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:45.933979Z","time spent":"497.866689ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"info","ts":"2026-03-31T00:41:46.431789Z","caller":"traceutil/trace.go:172","msg":"trace[505468781] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1095; }","duration":"471.91663ms","start":"2026-03-31T00:41:45.959854Z","end":"2026-03-31T00:41:46.431771Z","steps":["trace[505468781] 'agreement among raft nodes before linearized reading' (duration: 471.794088ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:46.432127Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:45.959828Z","time spent":"472.289606ms","remote":"127.0.0.1:48522","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-31T00:41:46.736003Z","caller":"traceutil/trace.go:172","msg":"trace[1326814101] linearizableReadLoop","detail":"{readStateIndex:1263; appliedIndex:1263; }","duration":"301.873627ms","start":"2026-03-31T00:41:46.434071Z","end":"2026-03-31T00:41:46.735944Z","steps":["trace[1326814101] 'read index received' (duration: 301.868177ms)","trace[1326814101] 'applied index is now lower than readState.Index' (duration: 4.43µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:46.736182Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"302.08744ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:46.736221Z","caller":"traceutil/trace.go:172","msg":"trace[1046824234] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1096; }","duration":"302.143431ms","start":"2026-03-31T00:41:46.434067Z","end":"2026-03-31T00:41:46.736210Z","steps":["trace[1046824234] 'agreement among raft nodes before linearized reading' (duration: 302.04898ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:46.736292Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:46.434053Z","time spent":"302.231152ms","remote":"127.0.0.1:48548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-31T00:41:46.737713Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"117.806769ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:46.737762Z","caller":"traceutil/trace.go:172","msg":"trace[28939052] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1096; }","duration":"117.85919ms","start":"2026-03-31T00:41:46.619892Z","end":"2026-03-31T00:41:46.737751Z","steps":["trace[28939052] 'agreement among raft nodes before linearized reading' (duration: 117.775298ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:47.901435Z","caller":"traceutil/trace.go:172","msg":"trace[932021289] linearizableReadLoop","detail":"{readStateIndex:1265; appliedIndex:1265; }","duration":"281.762104ms","start":"2026-03-31T00:41:47.619651Z","end":"2026-03-31T00:41:47.901413Z","steps":["trace[932021289] 'read index received' (duration: 281.755274ms)","trace[932021289] 'applied index is now lower than readState.Index' (duration: 5.39µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:47.901568Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"281.904766ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:47.901595Z","caller":"traceutil/trace.go:172","msg":"trace[580791981] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1097; }","duration":"281.946447ms","start":"2026-03-31T00:41:47.619642Z","end":"2026-03-31T00:41:47.901589Z","steps":["trace[580791981] 'agreement among raft nodes before linearized reading' (duration: 281.873956ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:47.902407Z","caller":"traceutil/trace.go:172","msg":"trace[679866866] transaction","detail":"{read_only:false; response_revision:1098; number_of_response:1; }","duration":"531.31372ms","start":"2026-03-31T00:41:47.371083Z","end":"2026-03-31T00:41:47.902396Z","steps":["trace[679866866] 'process raft request' (duration: 531.176477ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:47.902485Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:47.371058Z","time spent":"531.383291ms","remote":"127.0.0.1:48694","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":795,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-31T00:41:47.968711Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"304.355039ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-31T00:41:47.968792Z","caller":"traceutil/trace.go:172","msg":"trace[1716480933] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1102; }","duration":"304.44005ms","start":"2026-03-31T00:41:47.664334Z","end":"2026-03-31T00:41:47.968774Z","steps":["trace[1716480933] 'agreement among raft nodes before linearized reading' (duration: 304.247727ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:47.968998Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:47.664317Z","time spent":"304.651123ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":1,"response size":505,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:48.138021Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"130.167693ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-31T00:41:48.750004Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:49.250243Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:49.750757Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:50.251901Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:50.496073Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.95481083s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-31T00:41:50.752745Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:51.104905Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.485828283s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:51.104995Z","caller":"traceutil/trace.go:172","msg":"trace[1893876556] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1105; }","duration":"2.485936585s","start":"2026-03-31T00:41:48.619041Z","end":"2026-03-31T00:41:51.104978Z","steps":["trace[1893876556] 'range keys from in-memory index tree' (duration: 2.485778263s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:51.105114Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.497677654s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:51.105163Z","caller":"traceutil/trace.go:172","msg":"trace[426004065] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1105; }","duration":"1.497731105s","start":"2026-03-31T00:41:49.607419Z","end":"2026-03-31T00:41:51.105150Z","steps":["trace[426004065] 'range keys from in-memory index tree' (duration: 1.497635584s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:51.105973Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"609.504781ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-31T00:41:51.106095Z","caller":"traceutil/trace.go:172","msg":"trace[2085523576] transaction","detail":"{read_only:false; response_revision:1106; number_of_response:1; }","duration":"3.017711868s","start":"2026-03-31T00:41:48.088363Z","end":"2026-03-31T00:41:51.106075Z","steps":["trace[2085523576] 'process raft request' (duration: 2.408042716s)","trace[2085523576] 'compare' (duration: 608.268132ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:51.106281Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:48.088344Z","time spent":"3.017905892s","remote":"127.0.0.1:48694","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":696,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-31T00:41:51.253978Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:51.755041Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:51.960925Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000111676s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-31T00:41:51.961004Z","caller":"traceutil/trace.go:172","msg":"trace[293457183] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000203797s","start":"2026-03-31T00:41:49.960783Z","end":"2026-03-31T00:41:51.960987Z","steps":["trace[293457183] 'agreement among raft nodes before linearized reading' (duration: 2.000107386s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:51.961059Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:49.960766Z","time spent":"2.000276598s","remote":"127.0.0.1:48522","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/31 00:41:51 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-31T00:41:52.255192Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:52.673538Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"2.17723863s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-31T00:41:52.755824Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-31T00:41:52.767708Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"3119206cae914f30 [logterm: 2, index: 1281, vote: 0] ignored MsgPreVote from a11c20306510c1f7 [logterm: 2, index: 1281] at term 2: lease is not expired (remaining ticks: 9)"} {"level":"warn","ts":"2026-03-31T00:41:53.256895Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:53.757050Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:53.934168Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.258822164s","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-31T00:41:53.962262Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000414656s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-31T00:41:53.962347Z","caller":"traceutil/trace.go:172","msg":"trace[1276923951] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000522198s","start":"2026-03-31T00:41:51.961807Z","end":"2026-03-31T00:41:53.962329Z","steps":["trace[1276923951] 'agreement among raft nodes before linearized reading' (duration: 2.000410656s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:53.962396Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:51.961785Z","time spent":"2.000602708s","remote":"127.0.0.1:48548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-31T00:41:53.973871Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"3119206cae914f30 [logterm: 2, index: 1281, vote: 0] ignored MsgPreVote from a11c20306510c1f7 [logterm: 2, index: 1281] at term 2: lease is not expired (remaining ticks: 9)"} {"level":"info","ts":"2026-03-31T00:41:53.983187Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"3119206cae914f30 [logterm: 2, index: 1281, vote: 0] ignored MsgVote from a11c20306510c1f7 [logterm: 2, index: 1281] at term 2: lease is not expired (remaining ticks: 9)"} {"level":"warn","ts":"2026-03-31T00:41:54.262144Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:54.493427Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.999195573s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-31T00:41:54.493554Z","caller":"traceutil/trace.go:172","msg":"trace[273236303] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; }","duration":"4.999346565s","start":"2026-03-31T00:41:49.494189Z","end":"2026-03-31T00:41:54.493536Z","steps":["trace[273236303] 'agreement among raft nodes before linearized reading' (duration: 4.999190683s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:54.493611Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:49.494159Z","time spent":"4.999441316s","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:54.762557Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294573,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:54.964830Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"2.291129117s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-31T00:41:55.248088Z","caller":"etcdserver/v3_server.go:923","msg":"timed out waiting for read index response (local node might have slow network)","timeout":"7s"} {"level":"warn","ts":"2026-03-31T00:41:55.248260Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"7.001253781s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-75tdz\" limit:1 ","response":"","error":"etcdserver: request timed out"} {"level":"info","ts":"2026-03-31T00:41:55.248301Z","caller":"traceutil/trace.go:172","msg":"trace[1126708126] range","detail":"{range_begin:/registry/pods/kube-system/cilium-75tdz; range_end:; }","duration":"7.001311002s","start":"2026-03-31T00:41:48.246975Z","end":"2026-03-31T00:41:55.248286Z","steps":["trace[1126708126] 'agreement among raft nodes before linearized reading' (duration: 7.001234411s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:55.248431Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:48.246954Z","time spent":"7.001462224s","remote":"127.0.0.1:48942","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":0,"response size":0,"request content":"key:\"/registry/pods/kube-system/cilium-75tdz\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:55.748864Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294578,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-31T00:41:55.768626Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1233","msg":"3119206cae914f30 [logterm: 2, index: 1281, vote: 0] cast MsgPreVote for a11c20306510c1f7 [logterm: 2, index: 1281] at term 2"} {"level":"info","ts":"2026-03-31T00:41:55.772329Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1116","msg":"3119206cae914f30 [term: 2] received a MsgVote message with higher term from a11c20306510c1f7 [term: 4]"} {"level":"info","ts":"2026-03-31T00:41:55.772414Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"3119206cae914f30 became follower at term 4"} {"level":"info","ts":"2026-03-31T00:41:55.772441Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1233","msg":"3119206cae914f30 [logterm: 2, index: 1281, vote: 0] cast MsgVote for a11c20306510c1f7 [logterm: 2, index: 1281] at term 4"} {"level":"info","ts":"2026-03-31T00:41:55.772467Z","logger":"raft","caller":"v3@v3.6.0/node.go:376","msg":"raft.node: 3119206cae914f30 lost leader a11c20306510c1f7 at term 4"} {"level":"warn","ts":"2026-03-31T00:41:55.977368Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.99989947s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-31T00:41:55.977457Z","caller":"traceutil/trace.go:172","msg":"trace[1801463388] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.999995412s","start":"2026-03-31T00:41:53.977440Z","end":"2026-03-31T00:41:55.977435Z","steps":["trace[1801463388] 'agreement among raft nodes before linearized reading' (duration: 1.99989305s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:55.977522Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:53.977426Z","time spent":"2.000084323s","remote":"127.0.0.1:48522","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/31 00:41:55 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-31T00:41:56.249523Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294578,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-31T00:41:56.249652Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"3119206cae914f30 no leader at term 4; dropping index reading msg"} {"level":"warn","ts":"2026-03-31T00:41:56.750278Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294578,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-31T00:41:56.750369Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"3119206cae914f30 no leader at term 4; dropping index reading msg"} {"level":"warn","ts":"2026-03-31T00:41:57.020584Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.999045625s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-31T00:41:57.020663Z","caller":"traceutil/trace.go:172","msg":"trace[1635189265] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; }","duration":"4.999148886s","start":"2026-03-31T00:41:52.021496Z","end":"2026-03-31T00:41:57.020644Z","steps":["trace[1635189265] 'agreement among raft nodes before linearized reading' (duration: 4.999041115s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:57.020717Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:52.021475Z","time spent":"4.999231067s","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:57.250871Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294578,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-31T00:41:57.250966Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"3119206cae914f30 no leader at term 4; dropping index reading msg"} {"level":"info","ts":"2026-03-31T00:41:57.269057Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1233","msg":"3119206cae914f30 [logterm: 2, index: 1281, vote: a11c20306510c1f7] cast MsgPreVote for a11c20306510c1f7 [logterm: 2, index: 1281] at term 4"} {"level":"warn","ts":"2026-03-31T00:41:57.751960Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294578,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-31T00:41:57.752127Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"3119206cae914f30 no leader at term 4; dropping index reading msg"} {"level":"warn","ts":"2026-03-31T00:41:57.978269Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000206155s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-31T00:41:57.979026Z","caller":"traceutil/trace.go:172","msg":"trace[867181614] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000966187s","start":"2026-03-31T00:41:55.978038Z","end":"2026-03-31T00:41:57.979004Z","steps":["trace[867181614] 'agreement among raft nodes before linearized reading' (duration: 2.000203455s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:57.979187Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:55.978016Z","time spent":"2.001156589s","remote":"127.0.0.1:48548","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/31 00:41:57 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-31T00:41:58.010459Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1116","msg":"3119206cae914f30 [term: 4] received a MsgVote message with higher term from a11c20306510c1f7 [term: 5]"} {"level":"info","ts":"2026-03-31T00:41:58.010553Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"3119206cae914f30 became follower at term 5"} {"level":"info","ts":"2026-03-31T00:41:58.010579Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1233","msg":"3119206cae914f30 [logterm: 2, index: 1281, vote: 0] cast MsgVote for a11c20306510c1f7 [logterm: 2, index: 1281] at term 5"} {"level":"warn","ts":"2026-03-31T00:41:58.252163Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294578,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-31T00:41:58.252226Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"3119206cae914f30 no leader at term 5; dropping index reading msg"} {"level":"warn","ts":"2026-03-31T00:41:58.353396Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"2.580829596s","expected-duration":"1s"} {"level":"info","ts":"2026-03-31T00:41:58.372834Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: 3119206cae914f30 elected leader a11c20306510c1f7 at term 5"} {"level":"warn","ts":"2026-03-31T00:41:58.485988Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999639519s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-31T00:41:58.486094Z","caller":"traceutil/trace.go:172","msg":"trace[1607388514] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.99975859s","start":"2026-03-31T00:41:56.486316Z","end":"2026-03-31T00:41:58.486074Z","steps":["trace[1607388514] 'agreement among raft nodes before linearized reading' (duration: 1.999636719s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:58.486146Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:56.486294Z","time spent":"1.999843051s","remote":"127.0.0.1:48522","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/31 00:41:58 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-31T00:41:58.684423Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:51.683301Z","time spent":"7.001112392s","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-31T00:41:58.753058Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294578,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:59.104014Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:52.102984Z","time spent":"7.001022896s","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""} {"level":"warn","ts":"2026-03-31T00:41:59.253543Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294578,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:41:59.526940Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"5.014980631s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-31T00:41:59.527004Z","caller":"traceutil/trace.go:172","msg":"trace[1186859064] range","detail":"{range_begin:/registry/configmaps; range_end:; }","duration":"5.015058122s","start":"2026-03-31T00:41:54.511940Z","end":"2026-03-31T00:41:59.526998Z","steps":["trace[1186859064] 'agreement among raft nodes before linearized reading' (duration: 5.014978771s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.527027Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:54.511911Z","time spent":"5.015110843s","remote":"127.0.0.1:48756","response type":"/etcdserverpb.KV/Range","request count":0,"request size":24,"response count":0,"response size":0,"request content":"key:\"/registry/configmaps\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.527049Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"5.722249066s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-31T00:41:59.527060Z","caller":"traceutil/trace.go:172","msg":"trace[486621461] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices; range_end:; }","duration":"5.722260187s","start":"2026-03-31T00:41:53.804795Z","end":"2026-03-31T00:41:59.527055Z","steps":["trace[486621461] 'agreement among raft nodes before linearized reading' (duration: 5.722249387s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.527071Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:53.804772Z","time spent":"5.722295527s","remote":"127.0.0.1:53368","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":0,"request content":"key:\"/registry/apiregistration.k8s.io/apiservices\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.527088Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"8.422053136s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-31T00:41:59.527142Z","caller":"traceutil/trace.go:172","msg":"trace[548072316] range","detail":"{range_begin:; range_end:; }","duration":"8.422107046s","start":"2026-03-31T00:41:51.105031Z","end":"2026-03-31T00:41:59.527138Z","steps":["trace[548072316] 'agreement among raft nodes before linearized reading' (duration: 8.422053476s)"],"step_count":1} {"level":"error","ts":"2026-03-31T00:41:59.527211Z","caller":"etcdhttp/health.go:345","msg":"Health check error","path":"/readyz","reason":"[+]non_learner ok\n[+]data_corruption ok\n[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: leader changed\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHTTPEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:345\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2220\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-03-31T00:41:59.527468Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"8.835371638s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"warn","ts":"2026-03-31T00:41:59.527482Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"10.214667503s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourcequotas\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-31T00:41:59.527504Z","caller":"traceutil/trace.go:172","msg":"trace[1399583749] range","detail":"{range_begin:/registry/resourcequotas; range_end:; }","duration":"10.214692033s","start":"2026-03-31T00:41:49.312807Z","end":"2026-03-31T00:41:59.527499Z","steps":["trace[1399583749] 'agreement among raft nodes before linearized reading' (duration: 10.214667323s)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:41:59.527624Z","caller":"traceutil/trace.go:172","msg":"trace[1776146037] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; }","duration":"8.835394338s","start":"2026-03-31T00:41:50.692088Z","end":"2026-03-31T00:41:59.527483Z","steps":["trace[1776146037] 'agreement among raft nodes before linearized reading' (duration: 8.835372088s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.527610Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"9.700535749s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"warn","ts":"2026-03-31T00:41:59.527647Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:50.692060Z","time spent":"8.835581201s","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"info","ts":"2026-03-31T00:41:59.527655Z","caller":"traceutil/trace.go:172","msg":"trace[1833342838] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; }","duration":"9.700591749s","start":"2026-03-31T00:41:49.827056Z","end":"2026-03-31T00:41:59.527648Z","steps":["trace[1833342838] 'agreement among raft nodes before linearized reading' (duration: 9.700535089s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.527668Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"11.082100262s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"warn","ts":"2026-03-31T00:41:59.527681Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:49.827025Z","time spent":"9.70065003s","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":0,"response size":0,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-03-31T00:41:59.527679Z","caller":"traceutil/trace.go:172","msg":"trace[209265206] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; }","duration":"11.082111572s","start":"2026-03-31T00:41:48.445563Z","end":"2026-03-31T00:41:59.527675Z","steps":["trace[209265206] 'agreement among raft nodes before linearized reading' (duration: 11.082100832s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.527815Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:48.445538Z","time spent":"11.082270964s","remote":"127.0.0.1:49600","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":0,"response size":0,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true "} {"level":"warn","ts":"2026-03-31T00:41:59.527718Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"9.734261873s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.116\" limit:1 ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-31T00:41:59.528025Z","caller":"traceutil/trace.go:172","msg":"trace[916805752] range","detail":"{range_begin:/registry/masterleases/199.204.45.116; range_end:; }","duration":"9.734570848s","start":"2026-03-31T00:41:49.793448Z","end":"2026-03-31T00:41:59.528019Z","steps":["trace[916805752] 'agreement among raft nodes before linearized reading' (duration: 9.734262013s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.527861Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:49.312784Z","time spent":"10.214736934s","remote":"127.0.0.1:48722","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":0,"request content":"key:\"/registry/resourcequotas\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.528050Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:49.793424Z","time spent":"9.73461984s","remote":"127.0.0.1:48618","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":0,"response size":0,"request content":"key:\"/registry/masterleases/199.204.45.116\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.556984Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":5706233631807294581,"received-request-id":5706233631807294578} {"level":"warn","ts":"2026-03-31T00:41:59.865629Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"233.654036ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-31T00:41:59.867965Z","caller":"traceutil/trace.go:172","msg":"trace[1716890383] transaction","detail":"{read_only:false; response_revision:1121; number_of_response:1; }","duration":"5.894387033s","start":"2026-03-31T00:41:53.973262Z","end":"2026-03-31T00:41:59.867649Z","steps":["trace[1716890383] 'process raft request' (duration: 5.89414111s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.868117Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:53.973233Z","time spent":"5.894821119s","remote":"127.0.0.1:48694","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":779,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-31T00:41:59.871499Z","caller":"traceutil/trace.go:172","msg":"trace[1304833169] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1146; }","duration":"596.327907ms","start":"2026-03-31T00:41:59.275147Z","end":"2026-03-31T00:41:59.871475Z","steps":["trace[1304833169] 'process raft request' (duration: 596.146885ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.871589Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:59.274959Z","time spent":"596.58036ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":49,"response count":0,"response size":586,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-31T00:41:59.871675Z","caller":"traceutil/trace.go:172","msg":"trace[306648045] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1146; }","duration":"602.892848ms","start":"2026-03-31T00:41:59.268735Z","end":"2026-03-31T00:41:59.871627Z","steps":["trace[306648045] 'process raft request' (duration: 602.495383ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.871837Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:59.268716Z","time spent":"603.028971ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":69,"response count":0,"response size":725,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-31T00:41:59.873811Z","caller":"traceutil/trace.go:172","msg":"trace[939428988] linearizableReadLoop","detail":"{readStateIndex:1319; appliedIndex:1275; }","duration":"346.87904ms","start":"2026-03-31T00:41:59.526903Z","end":"2026-03-31T00:41:59.873782Z","steps":["trace[939428988] 'read index received' (duration: 31.178606ms)","trace[939428988] 'applied index is now lower than readState.Index' (duration: 315.699304ms)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:41:59.873918Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"366.037837ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csidrivers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:59.873957Z","caller":"traceutil/trace.go:172","msg":"trace[2097595992] range","detail":"{range_begin:/registry/csidrivers; range_end:; response_count:0; response_revision:1152; }","duration":"366.080588ms","start":"2026-03-31T00:41:59.507868Z","end":"2026-03-31T00:41:59.873949Z","steps":["trace[2097595992] 'agreement among raft nodes before linearized reading' (duration: 366.013117ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.873976Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:59.507854Z","time spent":"366.116618ms","remote":"127.0.0.1:49376","response type":"/etcdserverpb.KV/Range","request count":0,"request size":24,"response count":0,"response size":28,"request content":"key:\"/registry/csidrivers\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.874057Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.823743053s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 ","response":"range_response_count:1 size:509"} {"level":"info","ts":"2026-03-31T00:41:59.874113Z","caller":"traceutil/trace.go:172","msg":"trace[2106847335] range","detail":"{range_begin:/registry/leases/kube-system/kube-controller-manager; range_end:; response_count:1; response_revision:1152; }","duration":"1.823788824s","start":"2026-03-31T00:41:58.050301Z","end":"2026-03-31T00:41:59.874090Z","steps":["trace[2106847335] 'agreement among raft nodes before linearized reading' (duration: 1.823707953s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.874141Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:58.050287Z","time spent":"1.823847154s","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":532,"request content":"key:\"/registry/leases/kube-system/kube-controller-manager\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.874144Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.887529101s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:59.874181Z","caller":"traceutil/trace.go:172","msg":"trace[1887386458] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1152; }","duration":"1.887553941s","start":"2026-03-31T00:41:57.986609Z","end":"2026-03-31T00:41:59.874163Z","steps":["trace[1887386458] 'agreement among raft nodes before linearized reading' (duration: 1.8875029s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.874203Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:57.986593Z","time spent":"1.887605271s","remote":"127.0.0.1:48522","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-31T00:41:59.874483Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.601487278s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-75tdz\" limit:1 ","response":"range_response_count:1 size:18354"} {"level":"info","ts":"2026-03-31T00:41:59.874654Z","caller":"traceutil/trace.go:172","msg":"trace[175521918] range","detail":"{range_begin:/registry/pods/kube-system/cilium-75tdz; range_end:; response_count:1; response_revision:1152; }","duration":"4.60165363s","start":"2026-03-31T00:41:55.272969Z","end":"2026-03-31T00:41:59.874623Z","steps":["trace[175521918] 'agreement among raft nodes before linearized reading' (duration: 4.601370266s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.874741Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:55.272945Z","time spent":"4.601779552s","remote":"127.0.0.1:48942","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":18377,"request content":"key:\"/registry/pods/kube-system/cilium-75tdz\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.875048Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.520367321s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/clusterrolebindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:59.875083Z","caller":"traceutil/trace.go:172","msg":"trace[423288237] range","detail":"{range_begin:/registry/clusterrolebindings; range_end:; response_count:0; response_revision:1152; }","duration":"1.521173912s","start":"2026-03-31T00:41:58.353900Z","end":"2026-03-31T00:41:59.875074Z","steps":["trace[423288237] 'agreement among raft nodes before linearized reading' (duration: 1.520331751s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.875101Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:58.353888Z","time spent":"1.521206523s","remote":"127.0.0.1:49290","response type":"/etcdserverpb.KV/Range","request count":0,"request size":33,"response count":0,"response size":28,"request content":"key:\"/registry/clusterrolebindings\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.932874Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"375.528119ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-31T00:41:59.933866Z","caller":"traceutil/trace.go:172","msg":"trace[1322201996] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1162; }","duration":"376.528653ms","start":"2026-03-31T00:41:59.557321Z","end":"2026-03-31T00:41:59.933849Z","steps":["trace[1322201996] 'agreement among raft nodes before linearized reading' (duration: 375.499149ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.933913Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:59.557292Z","time spent":"376.609725ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.933165Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"378.835375ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:59.934087Z","caller":"traceutil/trace.go:172","msg":"trace[2081436611] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices; range_end:; response_count:0; response_revision:1162; }","duration":"379.766488ms","start":"2026-03-31T00:41:59.554311Z","end":"2026-03-31T00:41:59.934077Z","steps":["trace[2081436611] 'agreement among raft nodes before linearized reading' (duration: 378.796535ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.934160Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:59.554303Z","time spent":"379.843749ms","remote":"127.0.0.1:53368","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":0,"response size":28,"request content":"key:\"/registry/apiregistration.k8s.io/apiservices\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.933229Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"378.874267ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-31T00:41:59.934410Z","caller":"traceutil/trace.go:172","msg":"trace[2099026239] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:1162; }","duration":"380.031563ms","start":"2026-03-31T00:41:59.554348Z","end":"2026-03-31T00:41:59.934379Z","steps":["trace[2099026239] 'agreement among raft nodes before linearized reading' (duration: 378.849936ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.934465Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:59.554342Z","time spent":"380.106184ms","remote":"127.0.0.1:49094","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.933255Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"381.565614ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourcequotas\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:59.934562Z","caller":"traceutil/trace.go:172","msg":"trace[1446108814] range","detail":"{range_begin:/registry/resourcequotas; range_end:; response_count:0; response_revision:1162; }","duration":"382.892294ms","start":"2026-03-31T00:41:59.551655Z","end":"2026-03-31T00:41:59.934547Z","steps":["trace[1446108814] 'agreement among raft nodes before linearized reading' (duration: 381.502464ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.934617Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:59.551642Z","time spent":"382.957684ms","remote":"127.0.0.1:48722","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/resourcequotas\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.933327Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"405.012582ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:59.934829Z","caller":"traceutil/trace.go:172","msg":"trace[1631775630] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1162; }","duration":"406.503503ms","start":"2026-03-31T00:41:59.528308Z","end":"2026-03-31T00:41:59.934812Z","steps":["trace[1631775630] 'agreement among raft nodes before linearized reading' (duration: 405.000272ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.933365Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"379.108769ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.204.45.116\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-31T00:41:59.934905Z","caller":"traceutil/trace.go:172","msg":"trace[176485243] range","detail":"{range_begin:/registry/masterleases/199.204.45.116; range_end:; response_count:1; response_revision:1162; }","duration":"380.636951ms","start":"2026-03-31T00:41:59.554249Z","end":"2026-03-31T00:41:59.934886Z","steps":["trace[176485243] 'agreement among raft nodes before linearized reading' (duration: 379.004538ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.934957Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:59.554241Z","time spent":"380.698562ms","remote":"127.0.0.1:48618","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.204.45.116\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.933360Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"380.291316ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:41:59.935162Z","caller":"traceutil/trace.go:172","msg":"trace[867447392] range","detail":"{range_begin:/registry/configmaps; range_end:; response_count:0; response_revision:1162; }","duration":"382.047911ms","start":"2026-03-31T00:41:59.553055Z","end":"2026-03-31T00:41:59.935103Z","steps":["trace[867447392] 'agreement among raft nodes before linearized reading' (duration: 380.268036ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.935206Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:59.553046Z","time spent":"382.148104ms","remote":"127.0.0.1:48756","response type":"/etcdserverpb.KV/Range","request count":0,"request size":24,"response count":0,"response size":28,"request content":"key:\"/registry/configmaps\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:41:59.933764Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"381.870728ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true ","response":"range_response_count:128 size:10811"} {"level":"info","ts":"2026-03-31T00:41:59.936056Z","caller":"traceutil/trace.go:172","msg":"trace[1590294089] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; response_count:128; response_revision:1162; }","duration":"384.184051ms","start":"2026-03-31T00:41:59.551855Z","end":"2026-03-31T00:41:59.936039Z","steps":["trace[1590294089] 'agreement among raft nodes before linearized reading' (duration: 381.184899ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:41:59.936107Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:41:59.551847Z","time spent":"384.240882ms","remote":"127.0.0.1:49600","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":128,"response size":10834,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true "} {"level":"warn","ts":"2026-03-31T00:42:00.128621Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"160.952033ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-31T00:42:00.129032Z","caller":"traceutil/trace.go:172","msg":"trace[376192655] transaction","detail":"{read_only:false; response_revision:1170; number_of_response:1; }","duration":"160.992152ms","start":"2026-03-31T00:41:59.968023Z","end":"2026-03-31T00:42:00.129015Z","steps":["trace[376192655] 'process raft request' (duration: 160.83465ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:42:00.129026Z","caller":"traceutil/trace.go:172","msg":"trace[1413145789] transaction","detail":"{read_only:false; response_revision:1169; number_of_response:1; }","duration":"164.886237ms","start":"2026-03-31T00:41:59.964090Z","end":"2026-03-31T00:42:00.128977Z","steps":["trace[1413145789] 'process raft request' (duration: 164.642464ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:42:00.248425Z","caller":"traceutil/trace.go:172","msg":"trace[1230339276] linearizableReadLoop","detail":"{readStateIndex:1356; appliedIndex:1357; }","duration":"132.557156ms","start":"2026-03-31T00:42:00.115829Z","end":"2026-03-31T00:42:00.248386Z","steps":["trace[1230339276] 'read index received' (duration: 132.544956ms)","trace[1230339276] 'applied index is now lower than readState.Index' (duration: 10.38µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:42:00.248742Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"132.88937ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/kube-scheduler\" limit:1 ","response":"range_response_count:1 size:482"} {"level":"info","ts":"2026-03-31T00:42:00.248803Z","caller":"traceutil/trace.go:172","msg":"trace[2000270974] range","detail":"{range_begin:/registry/leases/kube-system/kube-scheduler; range_end:; response_count:1; response_revision:1178; }","duration":"132.971081ms","start":"2026-03-31T00:42:00.115817Z","end":"2026-03-31T00:42:00.248788Z","steps":["trace[2000270974] 'agreement among raft nodes before linearized reading' (duration: 132.728878ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:00.282091Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"151.026932ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:462"} {"level":"info","ts":"2026-03-31T00:42:00.282206Z","caller":"traceutil/trace.go:172","msg":"trace[1148382904] range","detail":"{range_begin:/registry/services/endpoints/default/kubernetes; range_end:; response_count:1; response_revision:1180; }","duration":"151.141604ms","start":"2026-03-31T00:42:00.131022Z","end":"2026-03-31T00:42:00.282164Z","steps":["trace[1148382904] 'agreement among raft nodes before linearized reading' (duration: 150.83616ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:42:00.573418Z","caller":"traceutil/trace.go:172","msg":"trace[1077738185] linearizableReadLoop","detail":"{readStateIndex:1361; appliedIndex:1362; }","duration":"285.597065ms","start":"2026-03-31T00:42:00.287799Z","end":"2026-03-31T00:42:00.573396Z","steps":["trace[1077738185] 'read index received' (duration: 285.586745ms)","trace[1077738185] 'applied index is now lower than readState.Index' (duration: 9.76µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:42:00.573631Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"285.826948ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/\" range_end:\"/registry/masterleases0\" ","response":"range_response_count:3 size:391"} {"level":"info","ts":"2026-03-31T00:42:00.573674Z","caller":"traceutil/trace.go:172","msg":"trace[226555761] range","detail":"{range_begin:/registry/masterleases/; range_end:/registry/masterleases0; response_count:3; response_revision:1182; }","duration":"285.880478ms","start":"2026-03-31T00:42:00.287782Z","end":"2026-03-31T00:42:00.573662Z","steps":["trace[226555761] 'agreement among raft nodes before linearized reading' (duration: 285.720586ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:42:00.573692Z","caller":"traceutil/trace.go:172","msg":"trace[1706368425] transaction","detail":"{read_only:false; response_revision:1183; number_of_response:1; }","duration":"223.41886ms","start":"2026-03-31T00:42:00.350262Z","end":"2026-03-31T00:42:00.573681Z","steps":["trace[1706368425] 'process raft request' (duration: 223.359879ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:00.575336Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"226.649495ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-75tdz\" limit:1 ","response":"range_response_count:1 size:18526"} {"level":"info","ts":"2026-03-31T00:42:00.575465Z","caller":"traceutil/trace.go:172","msg":"trace[1530168932] range","detail":"{range_begin:/registry/pods/kube-system/cilium-75tdz; range_end:; response_count:1; response_revision:1183; }","duration":"226.801947ms","start":"2026-03-31T00:42:00.348640Z","end":"2026-03-31T00:42:00.575442Z","steps":["trace[1530168932] 'agreement among raft nodes before linearized reading' (duration: 226.418621ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:01.811963Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"192.804321ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:42:01.812068Z","caller":"traceutil/trace.go:172","msg":"trace[1771515493] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1188; }","duration":"192.928673ms","start":"2026-03-31T00:42:01.619124Z","end":"2026-03-31T00:42:01.812053Z","steps":["trace[1771515493] 'range keys from in-memory index tree' (duration: 192.75634ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:42:02.109067Z","caller":"traceutil/trace.go:172","msg":"trace[1016175050] linearizableReadLoop","detail":"{readStateIndex:1375; appliedIndex:1375; }","duration":"311.989325ms","start":"2026-03-31T00:42:01.797039Z","end":"2026-03-31T00:42:02.109028Z","steps":["trace[1016175050] 'read index received' (duration: 311.983115ms)","trace[1016175050] 'applied index is now lower than readState.Index' (duration: 4.9µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:42:02.109290Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"312.207198ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:42:02.109382Z","caller":"traceutil/trace.go:172","msg":"trace[1595581231] range","detail":"{range_begin:/registry/endpointslices; range_end:; response_count:0; response_revision:1194; }","duration":"312.34634ms","start":"2026-03-31T00:42:01.797018Z","end":"2026-03-31T00:42:02.109364Z","steps":["trace[1595581231] 'agreement among raft nodes before linearized reading' (duration: 312.157977ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:02.109434Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:42:01.796992Z","time spent":"312.432861ms","remote":"127.0.0.1:49116","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/endpointslices\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:42:02.110639Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"298.488977ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:42:02.110692Z","caller":"traceutil/trace.go:172","msg":"trace[953003567] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1194; }","duration":"298.544718ms","start":"2026-03-31T00:42:01.812135Z","end":"2026-03-31T00:42:02.110680Z","steps":["trace[953003567] 'agreement among raft nodes before linearized reading' (duration: 298.471737ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:02.830588Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"133.599751ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-31T00:42:02.834591Z","caller":"traceutil/trace.go:172","msg":"trace[1206856472] linearizableReadLoop","detail":"{readStateIndex:1378; appliedIndex:1378; }","duration":"214.496947ms","start":"2026-03-31T00:42:02.620003Z","end":"2026-03-31T00:42:02.834500Z","steps":["trace[1206856472] 'read index received' (duration: 214.475807ms)","trace[1206856472] 'applied index is now lower than readState.Index' (duration: 19.22µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:42:02.834706Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"214.681449ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:42:02.834784Z","caller":"traceutil/trace.go:172","msg":"trace[687555375] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1196; }","duration":"214.766091ms","start":"2026-03-31T00:42:02.619995Z","end":"2026-03-31T00:42:02.834761Z","steps":["trace[687555375] 'agreement among raft nodes before linearized reading' (duration: 214.641359ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:42:02.951313Z","caller":"traceutil/trace.go:172","msg":"trace[1144779454] transaction","detail":"{read_only:false; response_revision:1198; number_of_response:1; }","duration":"228.977307ms","start":"2026-03-31T00:42:02.722317Z","end":"2026-03-31T00:42:02.951294Z","steps":["trace[1144779454] 'process raft request' (duration: 220.232825ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:05.460667Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":5706233631807294641,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-31T00:42:05.687881Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.068233396s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:42:05.688009Z","caller":"traceutil/trace.go:172","msg":"trace[438115507] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1224; }","duration":"1.068328198s","start":"2026-03-31T00:42:04.619621Z","end":"2026-03-31T00:42:05.687949Z","steps":["trace[438115507] 'range keys from in-memory index tree' (duration: 1.068183845s)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:05.689324Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"167.883674ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-31T00:42:05.694434Z","caller":"traceutil/trace.go:172","msg":"trace[1185007055] linearizableReadLoop","detail":"{readStateIndex:1411; appliedIndex:1412; }","duration":"734.014279ms","start":"2026-03-31T00:42:04.960403Z","end":"2026-03-31T00:42:05.694418Z","steps":["trace[1185007055] 'read index received' (duration: 734.007889ms)","trace[1185007055] 'applied index is now lower than readState.Index' (duration: 5.32µs)"],"step_count":2} {"level":"warn","ts":"2026-03-31T00:42:05.694569Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"734.172651ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:42:05.694603Z","caller":"traceutil/trace.go:172","msg":"trace[209803863] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1229; }","duration":"734.212222ms","start":"2026-03-31T00:42:04.960382Z","end":"2026-03-31T00:42:05.694594Z","steps":["trace[209803863] 'agreement among raft nodes before linearized reading' (duration: 734.145972ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:05.694632Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:42:04.960353Z","time spent":"734.272103ms","remote":"127.0.0.1:48522","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-31T00:42:05.696176Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"351.140437ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:42:05.696337Z","caller":"traceutil/trace.go:172","msg":"trace[679301515] range","detail":"{range_begin:/registry/serviceaccounts; range_end:; response_count:0; response_revision:1229; }","duration":"351.309309ms","start":"2026-03-31T00:42:05.344989Z","end":"2026-03-31T00:42:05.696299Z","steps":["trace[679301515] 'agreement among raft nodes before linearized reading' (duration: 351.097066ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:05.696376Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:42:05.344969Z","time spent":"351.396381ms","remote":"127.0.0.1:48982","response type":"/etcdserverpb.KV/Range","request count":0,"request size":29,"response count":0,"response size":28,"request content":"key:\"/registry/serviceaccounts\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:42:05.696395Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"317.009042ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/cilium-operator-75968c57df-thpfq.18a1c64432f41061\" limit:1 ","response":"range_response_count:1 size:869"} {"level":"info","ts":"2026-03-31T00:42:05.696430Z","caller":"traceutil/trace.go:172","msg":"trace[1382596834] range","detail":"{range_begin:/registry/events/kube-system/cilium-operator-75968c57df-thpfq.18a1c64432f41061; range_end:; response_count:1; response_revision:1229; }","duration":"317.044813ms","start":"2026-03-31T00:42:05.379374Z","end":"2026-03-31T00:42:05.696419Z","steps":["trace[1382596834] 'agreement among raft nodes before linearized reading' (duration: 316.905251ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:05.696454Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:42:05.379366Z","time spent":"317.081283ms","remote":"127.0.0.1:48694","response type":"/etcdserverpb.KV/Range","request count":0,"request size":82,"response count":1,"response size":892,"request content":"key:\"/registry/events/kube-system/cilium-operator-75968c57df-thpfq.18a1c64432f41061\" limit:1 "} {"level":"warn","ts":"2026-03-31T00:42:05.696795Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"318.062297ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-75tdz\" limit:1 ","response":"range_response_count:1 size:18890"} {"level":"info","ts":"2026-03-31T00:42:05.696828Z","caller":"traceutil/trace.go:172","msg":"trace[1170653675] range","detail":"{range_begin:/registry/pods/kube-system/cilium-75tdz; range_end:; response_count:1; response_revision:1229; }","duration":"318.095428ms","start":"2026-03-31T00:42:05.378723Z","end":"2026-03-31T00:42:05.696818Z","steps":["trace[1170653675] 'agreement among raft nodes before linearized reading' (duration: 317.897545ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:05.696881Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-31T00:42:05.378704Z","time spent":"318.168799ms","remote":"127.0.0.1:48942","response type":"/etcdserverpb.KV/Range","request count":0,"request size":43,"response count":1,"response size":18913,"request content":"key:\"/registry/pods/kube-system/cilium-75tdz\" limit:1 "} {"level":"info","ts":"2026-03-31T00:42:10.455350Z","caller":"traceutil/trace.go:172","msg":"trace[383034049] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1317; }","duration":"196.00695ms","start":"2026-03-31T00:42:10.259323Z","end":"2026-03-31T00:42:10.455330Z","steps":["trace[383034049] 'process raft request' (duration: 195.610555ms)"],"step_count":1} {"level":"info","ts":"2026-03-31T00:42:10.564810Z","caller":"traceutil/trace.go:172","msg":"trace[334743558] linearizableReadLoop","detail":"{readStateIndex:1536; appliedIndex:1536; }","duration":"254.123604ms","start":"2026-03-31T00:42:10.310660Z","end":"2026-03-31T00:42:10.564784Z","steps":["trace[334743558] 'read index received' (duration: 254.119614ms)","trace[334743558] 'applied index is now lower than readState.Index' (duration: 3.14µs)"],"step_count":2} {"level":"info","ts":"2026-03-31T00:42:10.564905Z","caller":"traceutil/trace.go:172","msg":"trace[1098389309] transaction","detail":"{read_only:false; number_of_response:1; response_revision:1318; }","duration":"294.174068ms","start":"2026-03-31T00:42:10.270710Z","end":"2026-03-31T00:42:10.564884Z","steps":["trace[1098389309] 'process raft request' (duration: 294.087747ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:10.565031Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"254.358627ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:438"} {"level":"info","ts":"2026-03-31T00:42:10.565066Z","caller":"traceutil/trace.go:172","msg":"trace[258430590] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:1318; }","duration":"254.410387ms","start":"2026-03-31T00:42:10.310648Z","end":"2026-03-31T00:42:10.565058Z","steps":["trace[258430590] 'agreement among raft nodes before linearized reading' (duration: 254.279996ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:10.567056Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"120.224831ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions/ciliumnetworkpolicies.cilium.io\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-31T00:42:10.567099Z","caller":"traceutil/trace.go:172","msg":"trace[1113511340] range","detail":"{range_begin:/registry/apiextensions.k8s.io/customresourcedefinitions/ciliumnetworkpolicies.cilium.io; range_end:; response_count:0; response_revision:1318; }","duration":"120.271092ms","start":"2026-03-31T00:42:10.446818Z","end":"2026-03-31T00:42:10.567089Z","steps":["trace[1113511340] 'agreement among raft nodes before linearized reading' (duration: 120.205801ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:10.567278Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"233.384768ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/namespaces/kube-system\" limit:1 ","response":"range_response_count:1 size:351"} {"level":"info","ts":"2026-03-31T00:42:10.567326Z","caller":"traceutil/trace.go:172","msg":"trace[1209528464] range","detail":"{range_begin:/registry/namespaces/kube-system; range_end:; response_count:1; response_revision:1318; }","duration":"233.419839ms","start":"2026-03-31T00:42:10.333884Z","end":"2026-03-31T00:42:10.567304Z","steps":["trace[1209528464] 'agreement among raft nodes before linearized reading' (duration: 233.114025ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:10.567338Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"138.084269ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/cilium-75tdz\" limit:1 ","response":"range_response_count:1 size:19904"} {"level":"info","ts":"2026-03-31T00:42:10.567770Z","caller":"traceutil/trace.go:172","msg":"trace[2105659713] range","detail":"{range_begin:/registry/pods/kube-system/cilium-75tdz; range_end:; response_count:1; response_revision:1318; }","duration":"138.297202ms","start":"2026-03-31T00:42:10.429086Z","end":"2026-03-31T00:42:10.567384Z","steps":["trace[2105659713] 'agreement among raft nodes before linearized reading' (duration: 137.993198ms)"],"step_count":1} {"level":"warn","ts":"2026-03-31T00:42:10.696538Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:57952","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:42:13.766343Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:57976","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:42:13.780540Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:57986","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:42:13.882974Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:58000","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:42:13.906240Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:58018","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:42:14.165249Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:58030","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-31T00:42:14.457022Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:58052","server-name":"","error":"EOF"}