{"level":"warn","ts":"2026-03-17T20:04:50.921603Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"warn","ts":"2026-03-17T20:04:50.921777Z","caller":"etcdmain/config.go:270","msg":"--snapshot-count is deprecated in 3.6 and will be decommissioned in 3.7."} {"level":"info","ts":"2026-03-17T20:04:50.921788Z","caller":"etcdmain/etcd.go:64","msg":"Running: ","args":["etcd","--advertise-client-urls=https://199.19.213.226:2379","--cert-file=/etc/kubernetes/pki/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/etcd","--feature-gates=InitialCorruptCheck=true","--initial-advertise-peer-urls=https://199.19.213.226:2380","--initial-cluster=controller-1=https://199.19.213.226:2380","--key-file=/etc/kubernetes/pki/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://199.19.213.226:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://199.19.213.226:2380","--name=controller-1","--peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/etc/kubernetes/pki/etcd/peer.key","--peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--snapshot-count=10000","--tls-min-version=TLS1.3","--trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt","--watch-progress-notify-interval=5s"]} {"level":"info","ts":"2026-03-17T20:04:50.921867Z","caller":"etcdmain/etcd.go:124","msg":"Initialize and start etcd server","data-dir":"/var/lib/etcd","dir-type":"empty"} {"level":"warn","ts":"2026-03-17T20:04:50.921886Z","caller":"embed/config.go:1209","msg":"Running http and grpc server on single port. This is not recommended for production."} {"level":"info","ts":"2026-03-17T20:04:50.921898Z","caller":"embed/etcd.go:138","msg":"configuring peer listeners","listen-peer-urls":["https://199.19.213.226:2380"]} {"level":"info","ts":"2026-03-17T20:04:50.921928Z","caller":"embed/etcd.go:544","msg":"starting with peer TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-17T20:04:50.922369Z","caller":"embed/etcd.go:146","msg":"configuring client listeners","listen-client-urls":["https://127.0.0.1:2379","https://199.19.213.226:2379"]} {"level":"info","ts":"2026-03-17T20:04:50.922484Z","caller":"embed/etcd.go:323","msg":"starting an etcd server","etcd-version":"3.6.4","git-sha":"5400cdc","go-version":"go1.23.11","go-os":"linux","go-arch":"amd64","max-cpu-set":8,"max-cpu-available":8,"member-initialized":false,"name":"controller-1","data-dir":"/var/lib/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://199.19.213.226:2380"],"listen-peer-urls":["https://199.19.213.226:2380"],"advertise-client-urls":["https://199.19.213.226:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.19.213.226:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"],"experimental-local-address":"","cors":["*"],"host-whitelist":["*"],"initial-cluster":"controller-1=https://199.19.213.226:2380","initial-cluster-state":"new","initial-cluster-token":"etcd-cluster","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"feature-gates":"InitialCorruptCheck=true","initial-corrupt-check":false,"corrupt-check-time-interval":"0s","compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","discovery-token":"","discovery-endpoints":"","discovery-dial-timeout":"2s","discovery-request-timeout":"5s","discovery-keepalive-time":"2s","discovery-keepalive-timeout":"6s","discovery-insecure-transport":true,"discovery-insecure-skip-tls-verify":false,"discovery-cert":"","discovery-key":"","discovery-cacert":"","discovery-user":"","downgrade-check-interval":"5s","max-learners":1,"v2-deprecation":"write-only"} {"level":"info","ts":"2026-03-17T20:04:50.923310Z","logger":"bbolt","caller":"backend/backend.go:203","msg":"Opening db file (/var/lib/etcd/member/snap/db) with mode -rw------- and with options: {Timeout: 0s, NoGrowSync: false, NoFreelistSync: true, PreLoadFreelist: false, FreelistType: hashmap, ReadOnly: false, MmapFlags: 8000, InitialMmapSize: 10737418240, PageSize: 0, NoSync: false, OpenFile: 0x0, Mlock: false, Logger: 0xc00011a478}"} {"level":"info","ts":"2026-03-17T20:04:50.930660Z","logger":"bbolt","caller":"bbolt@v1.4.2/db.go:321","msg":"Opening bbolt db (/var/lib/etcd/member/snap/db) successfully"} {"level":"info","ts":"2026-03-17T20:04:50.930711Z","caller":"storage/backend.go:80","msg":"opened backend db","path":"/var/lib/etcd/member/snap/db","took":"7.514366ms"} {"level":"info","ts":"2026-03-17T20:04:50.930736Z","caller":"etcdserver/bootstrap.go:220","msg":"restore consistentIndex","index":0} {"level":"info","ts":"2026-03-17T20:04:50.930746Z","caller":"etcdserver/bootstrap.go:94","msg":"bootstrapping cluster"} {"level":"info","ts":"2026-03-17T20:04:50.930784Z","caller":"etcdserver/bootstrap.go:101","msg":"bootstrapping storage"} {"level":"info","ts":"2026-03-17T20:04:50.945807Z","caller":"etcdserver/bootstrap.go:109","msg":"bootstrapping raft"} {"level":"info","ts":"2026-03-17T20:04:50.945959Z","caller":"etcdserver/bootstrap.go:499","msg":"starting local member","local-member-id":"c1a97aa766b87c7e","cluster-id":"34571fc6a0261e75"} {"level":"info","ts":"2026-03-17T20:04:50.945994Z","caller":"etcdserver/server.go:312","msg":"bootstrap successfully"} {"level":"info","ts":"2026-03-17T20:04:50.946145Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"c1a97aa766b87c7e switched to configuration voters=()"} {"level":"info","ts":"2026-03-17T20:04:50.946209Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"c1a97aa766b87c7e became follower at term 0"} {"level":"info","ts":"2026-03-17T20:04:50.946243Z","logger":"raft","caller":"v3@v3.6.0/raft.go:493","msg":"newRaft c1a97aa766b87c7e [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]"} {"level":"info","ts":"2026-03-17T20:04:50.946264Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"c1a97aa766b87c7e became follower at term 1"} {"level":"info","ts":"2026-03-17T20:04:50.946404Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"c1a97aa766b87c7e switched to configuration voters=(13954819779785686142)"} {"level":"warn","ts":"2026-03-17T20:04:50.987933Z","caller":"auth/store.go:1135","msg":"simple token is not cryptographically signed"} {"level":"info","ts":"2026-03-17T20:04:50.992301Z","caller":"mvcc/kvstore.go:408","msg":"kvstore restored","current-rev":1} {"level":"info","ts":"2026-03-17T20:04:50.995637Z","caller":"storage/quota.go:93","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"} {"level":"info","ts":"2026-03-17T20:04:50.997383Z","caller":"etcdserver/server.go:598","msg":"starting etcd server","local-member-id":"c1a97aa766b87c7e","local-server-version":"3.6.4","cluster-version":"to_be_decided"} {"level":"info","ts":"2026-03-17T20:04:50.998106Z","caller":"embed/etcd.go:766","msg":"starting with client TLS","tls-info":"cert = /etc/kubernetes/pki/etcd/server.crt, key = /etc/kubernetes/pki/etcd/server.key, client-cert=, client-key=, trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]} {"level":"info","ts":"2026-03-17T20:04:50.998097Z","caller":"etcdserver/server.go:483","msg":"started as single-node; fast-forwarding election ticks","local-member-id":"c1a97aa766b87c7e","forward-ticks":9,"forward-duration":"900ms","election-ticks":10,"election-timeout":"1s"} {"level":"info","ts":"2026-03-17T20:04:50.998257Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-17T20:04:50.998313Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-17T20:04:50.998325Z","caller":"fileutil/purge.go:49","msg":"started to purge file","dir":"/var/lib/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"} {"level":"info","ts":"2026-03-17T20:04:50.998360Z","caller":"embed/etcd.go:640","msg":"serving peer traffic","address":"199.19.213.226:2380"} {"level":"info","ts":"2026-03-17T20:04:50.998695Z","caller":"embed/etcd.go:611","msg":"cmux::serve","address":"199.19.213.226:2380"} {"level":"info","ts":"2026-03-17T20:04:50.998703Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"c1a97aa766b87c7e switched to configuration voters=(13954819779785686142)"} {"level":"info","ts":"2026-03-17T20:04:50.998956Z","caller":"embed/etcd.go:292","msg":"now serving peer/client/metrics","local-member-id":"c1a97aa766b87c7e","initial-advertise-peer-urls":["https://199.19.213.226:2380"],"listen-peer-urls":["https://199.19.213.226:2380"],"advertise-client-urls":["https://199.19.213.226:2379"],"listen-client-urls":["https://127.0.0.1:2379","https://199.19.213.226:2379"],"listen-metrics-urls":["http://127.0.0.1:2381"]} {"level":"info","ts":"2026-03-17T20:04:50.999020Z","caller":"embed/etcd.go:890","msg":"serving metrics","address":"http://127.0.0.1:2381"} {"level":"info","ts":"2026-03-17T20:04:50.999171Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"34571fc6a0261e75","local-member-id":"c1a97aa766b87c7e","added-peer-id":"c1a97aa766b87c7e","added-peer-peer-urls":["https://199.19.213.226:2380"],"added-peer-is-learner":false} {"level":"info","ts":"2026-03-17T20:04:51.047267Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"c1a97aa766b87c7e is starting a new election at term 1"} {"level":"info","ts":"2026-03-17T20:04:51.047380Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"c1a97aa766b87c7e became pre-candidate at term 1"} {"level":"info","ts":"2026-03-17T20:04:51.047452Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"c1a97aa766b87c7e received MsgPreVoteResp from c1a97aa766b87c7e at term 1"} {"level":"info","ts":"2026-03-17T20:04:51.047472Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"c1a97aa766b87c7e has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-17T20:04:51.047494Z","logger":"raft","caller":"v3@v3.6.0/raft.go:912","msg":"c1a97aa766b87c7e became candidate at term 2"} {"level":"info","ts":"2026-03-17T20:04:51.048192Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"c1a97aa766b87c7e received MsgVoteResp from c1a97aa766b87c7e at term 2"} {"level":"info","ts":"2026-03-17T20:04:51.048210Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"c1a97aa766b87c7e has received 1 MsgVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-17T20:04:51.048250Z","logger":"raft","caller":"v3@v3.6.0/raft.go:970","msg":"c1a97aa766b87c7e became leader at term 2"} {"level":"info","ts":"2026-03-17T20:04:51.048262Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: c1a97aa766b87c7e elected leader c1a97aa766b87c7e at term 2"} {"level":"info","ts":"2026-03-17T20:04:51.049268Z","caller":"etcdserver/server.go:1804","msg":"published local member to cluster through raft","local-member-id":"c1a97aa766b87c7e","local-member-attributes":"{Name:controller-1 ClientURLs:[https://199.19.213.226:2379]}","cluster-id":"34571fc6a0261e75","publish-timeout":"7s"} {"level":"info","ts":"2026-03-17T20:04:51.049417Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-17T20:04:51.049465Z","caller":"embed/serve.go:138","msg":"ready to serve client requests"} {"level":"info","ts":"2026-03-17T20:04:51.049495Z","caller":"etcdserver/server.go:2404","msg":"setting up initial cluster version using v3 API","cluster-version":"3.6"} {"level":"info","ts":"2026-03-17T20:04:51.049679Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} {"level":"info","ts":"2026-03-17T20:04:51.049740Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} {"level":"info","ts":"2026-03-17T20:04:51.050206Z","caller":"membership/cluster.go:682","msg":"set initial cluster version","cluster-id":"34571fc6a0261e75","local-member-id":"c1a97aa766b87c7e","cluster-version":"3.6"} {"level":"info","ts":"2026-03-17T20:04:51.050287Z","caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.6"} {"level":"info","ts":"2026-03-17T20:04:51.050339Z","caller":"etcdserver/server.go:2424","msg":"cluster version is updated","cluster-version":"3.6"} {"level":"info","ts":"2026-03-17T20:04:51.050454Z","caller":"version/monitor.go:116","msg":"cluster version differs from storage version.","cluster-version":"3.6.0","storage-version":"3.5.0"} {"level":"info","ts":"2026-03-17T20:04:51.050505Z","caller":"schema/migration.go:65","msg":"updated storage version","new-storage-version":"3.6.0"} {"level":"info","ts":"2026-03-17T20:04:51.051202Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"warn","ts":"2026-03-17T20:04:51.051607Z","caller":"v3rpc/grpc.go:52","msg":"etcdserver: failed to register grpc metrics","error":"duplicate metrics collector registration attempted"} {"level":"info","ts":"2026-03-17T20:04:51.051786Z","caller":"v3rpc/health.go:63","msg":"grpc service status changed","service":"","status":"SERVING"} {"level":"info","ts":"2026-03-17T20:04:51.057956Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"199.19.213.226:2379"} {"level":"info","ts":"2026-03-17T20:04:51.059474Z","caller":"embed/serve.go:283","msg":"serving client traffic securely","traffic":"grpc+http","address":"127.0.0.1:2379"} {"level":"warn","ts":"2026-03-17T20:04:51.566912Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46502","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.567118Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46488","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.581295Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46526","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.589286Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46540","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.610362Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46580","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.628602Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46596","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.635825Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46624","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.647907Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46644","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.656536Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46670","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.664036Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46676","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.670631Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46702","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.680774Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46714","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.694565Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46740","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.704787Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46748","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.714211Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46768","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.725059Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46794","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.732555Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46814","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.742883Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46836","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.753873Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46854","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.766033Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46874","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.774138Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46900","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.783123Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46928","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.822927Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46954","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.834801Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46966","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.846636Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:46988","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.856737Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47010","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.867488Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47022","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.875366Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47042","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.886802Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47070","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.899312Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47078","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.908277Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47092","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.918655Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47114","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.928412Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47154","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.935794Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47166","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.943197Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47180","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.950461Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47198","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.959848Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47212","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.967410Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47240","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.974725Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47272","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:51.995267Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47292","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.002552Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47318","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.011227Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47332","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.017239Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47364","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.025296Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47380","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.032523Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47392","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.045994Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47418","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.055497Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47436","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.064868Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47444","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.100816Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47468","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.116505Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47494","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.128479Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47518","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.141529Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47524","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.163487Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47542","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.178360Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47584","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.189024Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47600","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.207308Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47632","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.223043Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47638","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.237739Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47660","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.247697Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47670","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.259346Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47704","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:04:52.319335Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:47718","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-17T20:04:53.646236Z","caller":"traceutil/trace.go:172","msg":"trace[1572119940] transaction","detail":"{read_only:false; number_of_response:0; response_revision:71; }","duration":"102.514862ms","start":"2026-03-17T20:04:53.543706Z","end":"2026-03-17T20:04:53.646221Z","steps":["trace[1572119940] 'process raft request' (duration: 102.452721ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:04:59.180908Z","caller":"traceutil/trace.go:172","msg":"trace[836861570] linearizableReadLoop","detail":"{readStateIndex:326; appliedIndex:326; }","duration":"261.494931ms","start":"2026-03-17T20:04:58.919337Z","end":"2026-03-17T20:04:59.180831Z","steps":["trace[836861570] 'read index received' (duration: 261.482761ms)","trace[836861570] 'applied index is now lower than readState.Index' (duration: 10.24µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:04:59.181134Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"261.772388ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-17T20:04:59.181201Z","caller":"traceutil/trace.go:172","msg":"trace[725931720] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:309; }","duration":"261.850979ms","start":"2026-03-17T20:04:58.919331Z","end":"2026-03-17T20:04:59.181182Z","steps":["trace[725931720] 'agreement among raft nodes before linearized reading' (duration: 261.639855ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:04:59.181193Z","caller":"traceutil/trace.go:172","msg":"trace[664890178] transaction","detail":"{read_only:false; response_revision:310; number_of_response:1; }","duration":"319.699882ms","start":"2026-03-17T20:04:58.861474Z","end":"2026-03-17T20:04:59.181174Z","steps":["trace[664890178] 'process raft request' (duration: 319.574299ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:04:59.181556Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:04:58.861448Z","time spent":"319.808794ms","remote":"127.0.0.1:46914","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":201,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-17T20:04:59.184789Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"265.384655ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-17T20:04:59.184832Z","caller":"traceutil/trace.go:172","msg":"trace[1698598540] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:310; }","duration":"265.440587ms","start":"2026-03-17T20:04:58.919382Z","end":"2026-03-17T20:04:59.184822Z","steps":["trace[1698598540] 'agreement among raft nodes before linearized reading' (duration: 265.281523ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:04:59.184850Z","caller":"traceutil/trace.go:172","msg":"trace[882595370] transaction","detail":"{read_only:false; response_revision:311; number_of_response:1; }","duration":"256.852217ms","start":"2026-03-17T20:04:58.927963Z","end":"2026-03-17T20:04:59.184815Z","steps":["trace[882595370] 'process raft request' (duration: 256.738673ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:04:59.482883Z","caller":"traceutil/trace.go:172","msg":"trace[86072168] linearizableReadLoop","detail":"{readStateIndex:328; appliedIndex:328; }","duration":"295.798047ms","start":"2026-03-17T20:04:59.187054Z","end":"2026-03-17T20:04:59.482852Z","steps":["trace[86072168] 'read index received' (duration: 295.785987ms)","trace[86072168] 'applied index is now lower than readState.Index' (duration: 10.32µs)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:04:59.565156Z","caller":"traceutil/trace.go:172","msg":"trace[294465740] transaction","detail":"{read_only:false; response_revision:312; number_of_response:1; }","duration":"379.026385ms","start":"2026-03-17T20:04:59.186100Z","end":"2026-03-17T20:04:59.565127Z","steps":["trace[294465740] 'process raft request' (duration: 296.911925ms)","trace[294465740] 'compare' (duration: 82.005577ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:04:59.565169Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"378.03944ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts/kube-system/pv-protection-controller\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:04:59.565974Z","caller":"traceutil/trace.go:172","msg":"trace[1947229655] range","detail":"{range_begin:/registry/serviceaccounts/kube-system/pv-protection-controller; range_end:; response_count:0; response_revision:311; }","duration":"378.89794ms","start":"2026-03-17T20:04:59.187050Z","end":"2026-03-17T20:04:59.565948Z","steps":["trace[1947229655] 'agreement among raft nodes before linearized reading' (duration: 295.915729ms)","trace[1947229655] 'range keys from in-memory index tree' (duration: 82.081829ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:04:59.566131Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:04:59.187036Z","time spent":"379.034363ms","remote":"127.0.0.1:46914","response type":"/etcdserverpb.KV/Range","request count":0,"request size":66,"response count":0,"response size":28,"request content":"key:\"/registry/serviceaccounts/kube-system/pv-protection-controller\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:04:59.568167Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:04:59.186083Z","time spent":"379.784682ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":427,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:04:59.572034Z","caller":"traceutil/trace.go:172","msg":"trace[1756305434] transaction","detail":"{read_only:false; response_revision:313; number_of_response:1; }","duration":"382.884727ms","start":"2026-03-17T20:04:59.189137Z","end":"2026-03-17T20:04:59.572022Z","steps":["trace[1756305434] 'process raft request' (duration: 382.671191ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:04:59.572520Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:04:59.189120Z","time spent":"382.948278ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":423,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:05:05.065388Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"454.499518ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps/kube-public/cluster-info\" limit:1 ","response":"range_response_count:1 size:2304"} {"level":"warn","ts":"2026-03-17T20:05:05.065416Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"328.504237ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/kube-controller-manager-controller-1\" limit:1 ","response":"range_response_count:1 size:7368"} {"level":"info","ts":"2026-03-17T20:05:05.065478Z","caller":"traceutil/trace.go:172","msg":"trace[164412513] range","detail":"{range_begin:/registry/configmaps/kube-public/cluster-info; range_end:; response_count:1; response_revision:383; }","duration":"454.61102ms","start":"2026-03-17T20:05:04.610848Z","end":"2026-03-17T20:05:05.065459Z","steps":["trace[164412513] 'range keys from in-memory index tree' (duration: 454.280423ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:05:05.065538Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:05:04.610831Z","time spent":"454.694533ms","remote":"127.0.0.1:46692","response type":"/etcdserverpb.KV/Range","request count":0,"request size":49,"response count":1,"response size":2327,"request content":"key:\"/registry/configmaps/kube-public/cluster-info\" limit:1 "} {"level":"info","ts":"2026-03-17T20:05:05.065536Z","caller":"traceutil/trace.go:172","msg":"trace[1849994721] range","detail":"{range_begin:/registry/pods/kube-system/kube-controller-manager-controller-1; range_end:; response_count:1; response_revision:383; }","duration":"328.655921ms","start":"2026-03-17T20:05:04.736855Z","end":"2026-03-17T20:05:05.065511Z","steps":["trace[1849994721] 'range keys from in-memory index tree' (duration: 328.244451ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:05:05.065599Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:05:04.736831Z","time spent":"328.756864ms","remote":"127.0.0.1:46858","response type":"/etcdserverpb.KV/Range","request count":0,"request size":67,"response count":1,"response size":7391,"request content":"key:\"/registry/pods/kube-system/kube-controller-manager-controller-1\" limit:1 "} {"level":"info","ts":"2026-03-17T20:05:06.108233Z","caller":"traceutil/trace.go:172","msg":"trace[884190286] transaction","detail":"{read_only:false; response_revision:390; number_of_response:1; }","duration":"140.666222ms","start":"2026-03-17T20:05:05.967542Z","end":"2026-03-17T20:05:06.108208Z","steps":["trace[884190286] 'process raft request' (duration: 140.404716ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:05:10.521576Z","caller":"traceutil/trace.go:172","msg":"trace[696266328] transaction","detail":"{read_only:false; response_revision:403; number_of_response:1; }","duration":"209.040475ms","start":"2026-03-17T20:05:10.312513Z","end":"2026-03-17T20:05:10.521554Z","steps":["trace[696266328] 'process raft request' (duration: 208.879721ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:05:15.802289Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"284.362409ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:05:15.802308Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"178.904438ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:05:15.802357Z","caller":"traceutil/trace.go:172","msg":"trace[50146145] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:414; }","duration":"284.459271ms","start":"2026-03-17T20:05:15.517887Z","end":"2026-03-17T20:05:15.802347Z","steps":["trace[50146145] 'range keys from in-memory index tree' (duration: 284.241185ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:05:15.802398Z","caller":"traceutil/trace.go:172","msg":"trace[1545851097] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:414; }","duration":"179.032111ms","start":"2026-03-17T20:05:15.623347Z","end":"2026-03-17T20:05:15.802379Z","steps":["trace[1545851097] 'range keys from in-memory index tree' (duration: 178.833397ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:05:15.802418Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"126.931725ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-17T20:05:15.802468Z","caller":"traceutil/trace.go:172","msg":"trace[1262969349] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:414; }","duration":"126.999146ms","start":"2026-03-17T20:05:15.675458Z","end":"2026-03-17T20:05:15.802457Z","steps":["trace[1262969349] 'range keys from in-memory index tree' (duration: 126.704309ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:05:34.561197Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"c1a97aa766b87c7e switched to configuration voters=(13954819779785686142) learners=(9359522833427625171)"} {"level":"info","ts":"2026-03-17T20:05:34.561343Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"34571fc6a0261e75","local-member-id":"c1a97aa766b87c7e","added-peer-id":"81e3b46d999c74d3","added-peer-peer-urls":["https://199.19.213.206:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-03-17T20:05:34.561380Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:34.561410Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:34.561494Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:34.561481Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:34.561529Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3","remote-peer-urls":["https://199.19.213.206:2380"]} {"level":"info","ts":"2026-03-17T20:05:34.561548Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:34.561530Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:34.561579Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:34.561556Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"c1a97aa766b87c7e","raft-conf-change":"ConfChangeAddLearnerNode","raft-conf-change-node-id":"81e3b46d999c74d3"} {"level":"error","ts":"2026-03-17T20:05:34.614734Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"warn","ts":"2026-03-17T20:05:35.057456Z","caller":"etcdserver/cluster_util.go:259","msg":"failed to reach the peer URL","address":"https://199.19.213.206:2380/version","remote-member-id":"81e3b46d999c74d3","error":"Get \"https://199.19.213.206:2380/version\": dial tcp 199.19.213.206:2380: connect: connection refused"} {"level":"warn","ts":"2026-03-17T20:05:35.057525Z","caller":"etcdserver/cluster_util.go:160","msg":"failed to get version","remote-member-id":"81e3b46d999c74d3","error":"Get \"https://199.19.213.206:2380/version\": dial tcp 199.19.213.206:2380: connect: connection refused"} {"level":"error","ts":"2026-03-17T20:05:35.112454Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-17T20:05:35.619279Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-17T20:05:36.115620Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-17T20:05:36.618422Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"info","ts":"2026-03-17T20:05:36.778099Z","caller":"traceutil/trace.go:172","msg":"trace[1545727555] transaction","detail":"{read_only:false; response_revision:463; number_of_response:1; }","duration":"110.808071ms","start":"2026-03-17T20:05:36.667269Z","end":"2026-03-17T20:05:36.778077Z","steps":["trace[1545727555] 'process raft request' (duration: 110.680848ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:05:37.105438Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.19.213.206:58332","server-name":"","error":"EOF"} {"level":"error","ts":"2026-03-17T20:05:37.116155Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"error","ts":"2026-03-17T20:05:37.611018Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"info","ts":"2026-03-17T20:05:37.702577Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:37.702674Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:37.703266Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:37.711492Z","caller":"etcdserver/server.go:1838","msg":"sending merged snapshot","from":"c1a97aa766b87c7e","to":"81e3b46d999c74d3","bytes":798731,"size":"799 kB"} {"level":"info","ts":"2026-03-17T20:05:37.711606Z","caller":"rafthttp/snapshot_sender.go:82","msg":"sending database snapshot","snapshot-index":492,"remote-peer-id":"81e3b46d999c74d3","bytes":798731,"size":"799 kB"} {"level":"info","ts":"2026-03-17T20:05:37.713807Z","caller":"etcdserver/snapshot_merge.go:64","msg":"sent database snapshot to writer","bytes":790528,"size":"790 kB"} {"level":"info","ts":"2026-03-17T20:05:37.713940Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"c1a97aa766b87c7e","to":"81e3b46d999c74d3","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-17T20:05:37.714031Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:37.715885Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"c1a97aa766b87c7e","to":"81e3b46d999c74d3","stream-type":"stream Message"} {"level":"info","ts":"2026-03-17T20:05:37.715910Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:37.720035Z","caller":"rafthttp/snapshot_sender.go:131","msg":"sent database snapshot","snapshot-index":492,"remote-peer-id":"81e3b46d999c74d3","bytes":798731,"size":"799 kB"} {"level":"warn","ts":"2026-03-17T20:05:37.737935Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:05:37.738240Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3","error":"EOF"} {"level":"info","ts":"2026-03-17T20:05:37.747355Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"c1a97aa766b87c7e","to":"81e3b46d999c74d3","stream-type":"stream MsgApp v2"} {"level":"warn","ts":"2026-03-17T20:05:37.747378Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:37.747389Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:37.750991Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"c1a97aa766b87c7e","to":"81e3b46d999c74d3","stream-type":"stream Message"} {"level":"warn","ts":"2026-03-17T20:05:37.751027Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:37.751051Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:37.778170Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:37.778717Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:38.134934Z","caller":"traceutil/trace.go:172","msg":"trace[1576892997] transaction","detail":"{read_only:false; response_revision:467; number_of_response:1; }","duration":"160.217324ms","start":"2026-03-17T20:05:37.974702Z","end":"2026-03-17T20:05:38.134919Z","steps":["trace[1576892997] 'process raft request' (duration: 160.180463ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:05:38.135218Z","caller":"traceutil/trace.go:172","msg":"trace[620238885] transaction","detail":"{read_only:false; response_revision:466; number_of_response:1; }","duration":"160.50505ms","start":"2026-03-17T20:05:37.974702Z","end":"2026-03-17T20:05:38.135207Z","steps":["trace[620238885] 'process raft request' (duration: 160.092382ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:05:38.178439Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"c1a97aa766b87c7e switched to configuration voters=(9359522833427625171 13954819779785686142)"} {"level":"info","ts":"2026-03-17T20:05:38.178656Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"34571fc6a0261e75","local-member-id":"c1a97aa766b87c7e","promoted-member-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:38.178723Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"c1a97aa766b87c7e","raft-conf-change":"ConfChangeAddNode","raft-conf-change-node-id":"81e3b46d999c74d3"} {"level":"info","ts":"2026-03-17T20:05:42.805767Z","caller":"etcdserver/server.go:2246","msg":"skip compaction since there is an inflight snapshot"} {"level":"warn","ts":"2026-03-17T20:05:43.709582Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"191.368196ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:05:43.709656Z","caller":"traceutil/trace.go:172","msg":"trace[1497646723] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:477; }","duration":"191.458537ms","start":"2026-03-17T20:05:43.518182Z","end":"2026-03-17T20:05:43.709641Z","steps":["trace[1497646723] 'agreement among raft nodes before linearized reading' (duration: 24.152173ms)","trace[1497646723] 'range keys from in-memory index tree' (duration: 167.193332ms)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:05:46.378729Z","caller":"traceutil/trace.go:172","msg":"trace[995661783] linearizableReadLoop","detail":"{readStateIndex:515; appliedIndex:515; }","duration":"183.717677ms","start":"2026-03-17T20:05:46.194985Z","end":"2026-03-17T20:05:46.378702Z","steps":["trace[995661783] 'read index received' (duration: 183.707416ms)","trace[995661783] 'applied index is now lower than readState.Index' (duration: 8.491µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:05:46.379118Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"184.087033ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:436"} {"level":"info","ts":"2026-03-17T20:05:46.379244Z","caller":"traceutil/trace.go:172","msg":"trace[297399236] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:484; }","duration":"184.253555ms","start":"2026-03-17T20:05:46.194973Z","end":"2026-03-17T20:05:46.379226Z","steps":["trace[297399236] 'agreement among raft nodes before linearized reading' (duration: 183.829598ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:05:46.379136Z","caller":"traceutil/trace.go:172","msg":"trace[1443445591] transaction","detail":"{read_only:false; response_revision:485; number_of_response:1; }","duration":"205.607581ms","start":"2026-03-17T20:05:46.173496Z","end":"2026-03-17T20:05:46.379103Z","steps":["trace[1443445591] 'process raft request' (duration: 205.357987ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:05:47.161545Z","caller":"traceutil/trace.go:172","msg":"trace[131854912] transaction","detail":"{read_only:false; response_revision:487; number_of_response:1; }","duration":"344.151038ms","start":"2026-03-17T20:05:46.817379Z","end":"2026-03-17T20:05:47.161530Z","steps":["trace[131854912] 'process raft request' (duration: 344.049526ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:05:47.161686Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:05:46.817326Z","time spent":"344.291919ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":494,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:05:47.373424Z","caller":"traceutil/trace.go:172","msg":"trace[1120650511] transaction","detail":"{read_only:false; response_revision:488; number_of_response:1; }","duration":"225.547863ms","start":"2026-03-17T20:05:47.147861Z","end":"2026-03-17T20:05:47.373409Z","steps":["trace[1120650511] 'process raft request' (duration: 225.435752ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:05:47.653570Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"135.298679ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:05:47.653687Z","caller":"traceutil/trace.go:172","msg":"trace[1542501782] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:488; }","duration":"135.435421ms","start":"2026-03-17T20:05:47.518227Z","end":"2026-03-17T20:05:47.653662Z","steps":["trace[1542501782] 'range keys from in-memory index tree' (duration: 135.231417ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:05:47.653701Z","caller":"traceutil/trace.go:172","msg":"trace[89323466] transaction","detail":"{read_only:false; response_revision:489; number_of_response:1; }","duration":"207.351535ms","start":"2026-03-17T20:05:47.446324Z","end":"2026-03-17T20:05:47.653675Z","steps":["trace[89323466] 'process raft request' (duration: 167.648419ms)","trace[89323466] 'compare' (duration: 39.531313ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:01.855719Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"8.243476ms"} {"level":"info","ts":"2026-03-17T20:06:01.856590Z","caller":"traceutil/trace.go:172","msg":"trace[1154976238] transaction","detail":"{read_only:false; response_revision:554; number_of_response:1; }","duration":"147.070989ms","start":"2026-03-17T20:06:01.709506Z","end":"2026-03-17T20:06:01.856577Z","steps":["trace[1154976238] 'process raft request' (duration: 146.968228ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:01.856914Z","caller":"traceutil/trace.go:172","msg":"trace[1747232899] linearizableReadLoop","detail":"{readStateIndex:591; appliedIndex:592; }","duration":"120.282137ms","start":"2026-03-17T20:06:01.736587Z","end":"2026-03-17T20:06:01.856869Z","steps":["trace[1747232899] 'read index received' (duration: 120.274536ms)","trace[1747232899] 'applied index is now lower than readState.Index' (duration: 6.33µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:01.857097Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"120.49537ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/clusterrolebindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:01.857134Z","caller":"traceutil/trace.go:172","msg":"trace[1578059803] range","detail":"{range_begin:/registry/clusterrolebindings; range_end:; response_count:0; response_revision:554; }","duration":"120.561411ms","start":"2026-03-17T20:06:01.736562Z","end":"2026-03-17T20:06:01.857123Z","steps":["trace[1578059803] 'agreement among raft nodes before linearized reading' (duration: 120.427239ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:03.110753Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931979608,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:06:03.135945Z","caller":"traceutil/trace.go:172","msg":"trace[1664501867] linearizableReadLoop","detail":"{readStateIndex:592; appliedIndex:592; }","duration":"525.824255ms","start":"2026-03-17T20:06:02.610087Z","end":"2026-03-17T20:06:03.135912Z","steps":["trace[1664501867] 'read index received' (duration: 525.814385ms)","trace[1664501867] 'applied index is now lower than readState.Index' (duration: 7.99µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:03.136205Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"526.08808ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/podtemplates\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:03.136256Z","caller":"traceutil/trace.go:172","msg":"trace[1737434408] range","detail":"{range_begin:/registry/podtemplates; range_end:; response_count:0; response_revision:554; }","duration":"526.170731ms","start":"2026-03-17T20:06:02.610071Z","end":"2026-03-17T20:06:03.136242Z","steps":["trace[1737434408] 'agreement among raft nodes before linearized reading' (duration: 526.029599ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:03.136325Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:02.609995Z","time spent":"526.316684ms","remote":"127.0.0.1:46730","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":28,"request content":"key:\"/registry/podtemplates\" limit:1 "} {"level":"info","ts":"2026-03-17T20:06:03.136290Z","caller":"traceutil/trace.go:172","msg":"trace[19480931] transaction","detail":"{read_only:false; response_revision:555; number_of_response:1; }","duration":"557.261636ms","start":"2026-03-17T20:06:02.579007Z","end":"2026-03-17T20:06:03.136268Z","steps":["trace[19480931] 'process raft request' (duration: 557.053163ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:03.136499Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:02.578991Z","time spent":"557.416368ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":427,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:03.137142Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"512.018153ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:03.137253Z","caller":"traceutil/trace.go:172","msg":"trace[2099077433] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:555; }","duration":"512.136825ms","start":"2026-03-17T20:06:02.625097Z","end":"2026-03-17T20:06:03.137233Z","steps":["trace[2099077433] 'agreement among raft nodes before linearized reading' (duration: 511.973602ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:03.137298Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:02.625068Z","time spent":"512.220565ms","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:06:03.137430Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"487.363288ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"warn","ts":"2026-03-17T20:06:03.137549Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"509.989619ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/daemonsets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:03.137570Z","caller":"traceutil/trace.go:172","msg":"trace[2147430022] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:555; }","duration":"487.504971ms","start":"2026-03-17T20:06:02.650056Z","end":"2026-03-17T20:06:03.137561Z","steps":["trace[2147430022] 'agreement among raft nodes before linearized reading' (duration: 487.171224ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:03.137588Z","caller":"traceutil/trace.go:172","msg":"trace[1147198467] range","detail":"{range_begin:/registry/daemonsets; range_end:; response_count:0; response_revision:555; }","duration":"510.02817ms","start":"2026-03-17T20:06:02.627548Z","end":"2026-03-17T20:06:03.137576Z","steps":["trace[1147198467] 'agreement among raft nodes before linearized reading' (duration: 509.965319ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:03.137598Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:02.650043Z","time spent":"487.548411ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:03.137621Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:02.627539Z","time spent":"510.07483ms","remote":"127.0.0.1:47508","response type":"/etcdserverpb.KV/Range","request count":0,"request size":24,"response count":0,"response size":28,"request content":"key:\"/registry/daemonsets\" limit:1 "} {"level":"info","ts":"2026-03-17T20:06:03.266729Z","caller":"traceutil/trace.go:172","msg":"trace[1826584136] transaction","detail":"{read_only:false; response_revision:556; number_of_response:1; }","duration":"125.08491ms","start":"2026-03-17T20:06:03.141621Z","end":"2026-03-17T20:06:03.266706Z","steps":["trace[1826584136] 'process raft request' (duration: 124.946517ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:05.938590Z","caller":"etcdserver/server.go:2246","msg":"skip compaction since there is an inflight snapshot"} {"level":"info","ts":"2026-03-17T20:06:07.332358Z","caller":"traceutil/trace.go:172","msg":"trace[474768545] transaction","detail":"{read_only:false; response_revision:568; number_of_response:1; }","duration":"117.672474ms","start":"2026-03-17T20:06:07.214661Z","end":"2026-03-17T20:06:07.332333Z","steps":["trace[474768545] 'process raft request' (duration: 116.820049ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:07.720707Z","caller":"etcdserver/server.go:1856","msg":"sent merged snapshot","from":"c1a97aa766b87c7e","to":"81e3b46d999c74d3","bytes":798731,"size":"799 kB","took":"30.009198063s"} {"level":"info","ts":"2026-03-17T20:06:08.015163Z","caller":"traceutil/trace.go:172","msg":"trace[1158205059] transaction","detail":"{read_only:false; response_revision:571; number_of_response:1; }","duration":"127.824563ms","start":"2026-03-17T20:06:07.887322Z","end":"2026-03-17T20:06:08.015146Z","steps":["trace[1158205059] 'process raft request' (duration: 127.717511ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:18.344278Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"c1a97aa766b87c7e switched to configuration voters=(9359522833427625171 13954819779785686142) learners=(14327884284320208212)"} {"level":"info","ts":"2026-03-17T20:06:18.344476Z","caller":"membership/cluster.go:424","msg":"added member","cluster-id":"34571fc6a0261e75","local-member-id":"c1a97aa766b87c7e","added-peer-id":"c6d6ded81dced554","added-peer-peer-urls":["https://199.204.45.72:2380"],"added-peer-is-learner":true} {"level":"info","ts":"2026-03-17T20:06:18.344511Z","caller":"rafthttp/peer.go:133","msg":"starting remote peer","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:18.344543Z","caller":"rafthttp/pipeline.go:72","msg":"started HTTP pipelining with remote peer","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:18.344970Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:18.350368Z","caller":"rafthttp/stream.go:168","msg":"started stream writer with remote peer","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:18.350639Z","caller":"rafthttp/peer.go:137","msg":"started remote peer","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:18.350671Z","caller":"rafthttp/transport.go:317","msg":"added remote peer","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554","remote-peer-urls":["https://199.204.45.72:2380"]} {"level":"info","ts":"2026-03-17T20:06:18.350676Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:18.350941Z","caller":"rafthttp/stream.go:394","msg":"started stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"error","ts":"2026-03-17T20:06:18.402569Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*peerMemberPromoteHandler).ServeHTTP\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/peer.go:140\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-03-17T20:06:18.402650Z","caller":"etcdhttp/peer.go:152","msg":"failed to promote a member","member-id":"c6d6ded81dced554","error":"etcdserver: can only promote a learner member which is in sync with leader"} {"level":"warn","ts":"2026-03-17T20:06:18.898567Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.72:39162","server-name":"","error":"EOF"} {"level":"error","ts":"2026-03-17T20:06:18.899356Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"warn","ts":"2026-03-17T20:06:19.392491Z","caller":"etcdserver/cluster_util.go:259","msg":"failed to reach the peer URL","address":"https://199.204.45.72:2380/version","remote-member-id":"c6d6ded81dced554","error":"Get \"https://199.204.45.72:2380/version\": dial tcp 199.204.45.72:2380: connect: connection refused"} {"level":"warn","ts":"2026-03-17T20:06:19.392553Z","caller":"etcdserver/cluster_util.go:160","msg":"failed to get version","remote-member-id":"c6d6ded81dced554","error":"Get \"https://199.204.45.72:2380/version\": dial tcp 199.204.45.72:2380: connect: connection refused"} {"level":"warn","ts":"2026-03-17T20:06:19.397045Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.72:39174","server-name":"","error":"EOF"} {"level":"error","ts":"2026-03-17T20:06:19.397701Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*peerMemberPromoteHandler).ServeHTTP\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/peer.go:140\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-03-17T20:06:19.397756Z","caller":"etcdhttp/peer.go:152","msg":"failed to promote a member","member-id":"c6d6ded81dced554","error":"etcdserver: can only promote a learner member which is in sync with leader"} {"level":"error","ts":"2026-03-17T20:06:19.895069Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"warn","ts":"2026-03-17T20:06:20.400851Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"199.204.45.72:39200","server-name":"","error":"EOF"} {"level":"error","ts":"2026-03-17T20:06:20.401671Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*peerMemberPromoteHandler).ServeHTTP\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/peer.go:140\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"warn","ts":"2026-03-17T20:06:20.401753Z","caller":"etcdhttp/peer.go:152","msg":"failed to promote a member","member-id":"c6d6ded81dced554","error":"etcdserver: can only promote a learner member which is in sync with leader"} {"level":"error","ts":"2026-03-17T20:06:20.897353Z","caller":"etcdserver/server.go:1585","msg":"rejecting promote learner: learner is not ready","learner-ready-percent":0,"ready-percent-threshold":0.9,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).isLearnerReady\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1585\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).mayPromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1526\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).promoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1498\ngo.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).PromoteMember\n\tgo.etcd.io/etcd/server/v3/etcdserver/server.go:1450\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*ClusterServer).MemberPromote\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/member.go:101\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler.func1\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7432\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.(*ServerMetrics).UnaryServerInterceptor.UnaryServerInterceptor.func12\n\tgithub.com/grpc-ecosystem/go-grpc-middleware/v2@v2.1.0/interceptors/server.go:22\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func5\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:74\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1217\ngo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func4\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/v3rpc/interceptor.go:81\ngoogle.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1208\ngo.etcd.io/etcd/api/v3/etcdserverpb._Cluster_MemberPromote_Handler\n\tgo.etcd.io/etcd/api/v3@v3.6.4/etcdserverpb/rpc.pb.go:7434\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1405\ngoogle.golang.org/grpc.(*Server).handleStream\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1815\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\tgoogle.golang.org/grpc@v1.71.1/server.go:1035"} {"level":"info","ts":"2026-03-17T20:06:21.223517Z","caller":"rafthttp/peer_status.go:53","msg":"peer became active","peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:21.234331Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"c1a97aa766b87c7e","to":"c6d6ded81dced554","stream-type":"stream Message"} {"level":"info","ts":"2026-03-17T20:06:21.234374Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:21.235694Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"c1a97aa766b87c7e","to":"c6d6ded81dced554","stream-type":"stream MsgApp v2"} {"level":"info","ts":"2026-03-17T20:06:21.235735Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:21.241637Z","caller":"etcdserver/server.go:1838","msg":"sending merged snapshot","from":"c1a97aa766b87c7e","to":"c6d6ded81dced554","bytes":1032751,"size":"1.0 MB"} {"level":"info","ts":"2026-03-17T20:06:21.241847Z","caller":"rafthttp/snapshot_sender.go:82","msg":"sending database snapshot","snapshot-index":657,"remote-peer-id":"c6d6ded81dced554","bytes":1032751,"size":"1.0 MB"} {"level":"info","ts":"2026-03-17T20:06:21.245670Z","caller":"etcdserver/snapshot_merge.go:64","msg":"sent database snapshot to writer","bytes":1024000,"size":"1.0 MB"} {"level":"info","ts":"2026-03-17T20:06:21.256841Z","caller":"rafthttp/snapshot_sender.go:131","msg":"sent database snapshot","snapshot-index":657,"remote-peer-id":"c6d6ded81dced554","bytes":1032751,"size":"1.0 MB"} {"level":"info","ts":"2026-03-17T20:06:21.260670Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:21.264042Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"warn","ts":"2026-03-17T20:06:21.268354Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:06:21.268436Z","caller":"rafthttp/stream.go:420","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554","error":"EOF"} {"level":"info","ts":"2026-03-17T20:06:21.279521Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"c1a97aa766b87c7e","to":"c6d6ded81dced554","stream-type":"stream MsgApp v2"} {"level":"warn","ts":"2026-03-17T20:06:21.279548Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:21.279559Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:21.284252Z","caller":"rafthttp/stream.go:248","msg":"set message encoder","from":"c1a97aa766b87c7e","to":"c6d6ded81dced554","stream-type":"stream Message"} {"level":"warn","ts":"2026-03-17T20:06:21.284478Z","caller":"rafthttp/stream.go:264","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:21.284543Z","caller":"rafthttp/stream.go:273","msg":"established TCP streaming connection with remote peer","stream-writer-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:21.362634Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:21.363871Z","caller":"rafthttp/stream.go:411","msg":"established TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"c1a97aa766b87c7e","remote-peer-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:21.401563Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1981","msg":"c1a97aa766b87c7e switched to configuration voters=(9359522833427625171 13954819779785686142 14327884284320208212)"} {"level":"info","ts":"2026-03-17T20:06:21.401809Z","caller":"membership/cluster.go:550","msg":"promote member","cluster-id":"34571fc6a0261e75","local-member-id":"c1a97aa766b87c7e","promoted-member-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:21.401876Z","caller":"etcdserver/server.go:1752","msg":"applied a configuration change through raft","local-member-id":"c1a97aa766b87c7e","raft-conf-change":"ConfChangeAddNode","raft-conf-change-node-id":"c6d6ded81dced554"} {"level":"info","ts":"2026-03-17T20:06:25.618491Z","caller":"etcdserver/server.go:2246","msg":"skip compaction since there is an inflight snapshot"} {"level":"warn","ts":"2026-03-17T20:06:32.617205Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"369.739622ms"} {"level":"warn","ts":"2026-03-17T20:06:32.617297Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"369.840824ms"} {"level":"info","ts":"2026-03-17T20:06:32.618490Z","caller":"traceutil/trace.go:172","msg":"trace[799564729] linearizableReadLoop","detail":"{readStateIndex:767; appliedIndex:767; }","duration":"100.151435ms","start":"2026-03-17T20:06:32.518296Z","end":"2026-03-17T20:06:32.618448Z","steps":["trace[799564729] 'read index received' (duration: 100.144685ms)","trace[799564729] 'applied index is now lower than readState.Index' (duration: 5.45µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:32.746793Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"228.540979ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:32.746887Z","caller":"traceutil/trace.go:172","msg":"trace[507019752] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:711; }","duration":"228.639191ms","start":"2026-03-17T20:06:32.518230Z","end":"2026-03-17T20:06:32.746869Z","steps":["trace[507019752] 'agreement among raft nodes before linearized reading' (duration: 100.350128ms)","trace[507019752] 'range keys from in-memory index tree' (duration: 128.174101ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:32.747042Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"129.582083ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-17T20:06:32.747141Z","caller":"traceutil/trace.go:172","msg":"trace[2017355705] linearizableReadLoop","detail":"{readStateIndex:768; appliedIndex:767; }","duration":"119.725063ms","start":"2026-03-17T20:06:32.627405Z","end":"2026-03-17T20:06:32.747130Z","steps":["trace[2017355705] 'read index received' (duration: 740.271µs)","trace[2017355705] 'applied index is now lower than readState.Index' (duration: 118.983832ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:32.747191Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"119.788554ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:32.747213Z","caller":"traceutil/trace.go:172","msg":"trace[190269038] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:712; }","duration":"119.811784ms","start":"2026-03-17T20:06:32.627396Z","end":"2026-03-17T20:06:32.747208Z","steps":["trace[190269038] 'agreement among raft nodes before linearized reading' (duration: 119.765904ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:32.747362Z","caller":"traceutil/trace.go:172","msg":"trace[307478131] transaction","detail":"{read_only:false; response_revision:712; number_of_response:1; }","duration":"620.949273ms","start":"2026-03-17T20:06:32.126241Z","end":"2026-03-17T20:06:32.747191Z","steps":["trace[307478131] 'process raft request' (duration: 491.133917ms)","trace[307478131] 'compare' (duration: 129.332668ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:32.747527Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:32.126215Z","time spent":"621.228266ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":467,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:33.302231Z","caller":"traceutil/trace.go:172","msg":"trace[1510212650] transaction","detail":"{read_only:false; response_revision:714; number_of_response:1; }","duration":"211.915777ms","start":"2026-03-17T20:06:33.090286Z","end":"2026-03-17T20:06:33.302202Z","steps":["trace[1510212650] 'process raft request' (duration: 171.422352ms)","trace[1510212650] 'compare' (duration: 40.313053ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:34.675260Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"156.980037ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:34.675732Z","caller":"traceutil/trace.go:172","msg":"trace[467339631] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:722; }","duration":"157.463174ms","start":"2026-03-17T20:06:34.518257Z","end":"2026-03-17T20:06:34.675720Z","steps":["trace[467339631] 'range keys from in-memory index tree' (duration: 156.940816ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:35.608292Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"102.07437ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-17T20:06:35.608362Z","caller":"traceutil/trace.go:172","msg":"trace[261845917] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:728; }","duration":"103.054136ms","start":"2026-03-17T20:06:35.505296Z","end":"2026-03-17T20:06:35.608350Z","steps":["trace[261845917] 'range keys from in-memory index tree' (duration: 100.060531ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:36.356793Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"9.202041ms"} {"level":"warn","ts":"2026-03-17T20:06:36.356984Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"9.402364ms"} {"level":"warn","ts":"2026-03-17T20:06:37.030579Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931979901,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:06:37.044792Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"487.787536ms"} {"level":"warn","ts":"2026-03-17T20:06:37.044850Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"487.856617ms"} {"level":"info","ts":"2026-03-17T20:06:37.235527Z","caller":"traceutil/trace.go:172","msg":"trace[786268872] linearizableReadLoop","detail":"{readStateIndex:790; appliedIndex:791; }","duration":"718.246497ms","start":"2026-03-17T20:06:36.517254Z","end":"2026-03-17T20:06:37.235500Z","steps":["trace[786268872] 'read index received' (duration: 718.240517ms)","trace[786268872] 'applied index is now lower than readState.Index' (duration: 4.36µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:37.622093Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.104863873s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:37.622155Z","caller":"traceutil/trace.go:172","msg":"trace[2018551126] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:732; }","duration":"1.104932974s","start":"2026-03-17T20:06:36.517211Z","end":"2026-03-17T20:06:37.622144Z","steps":["trace[2018551126] 'agreement among raft nodes before linearized reading' (duration: 718.481161ms)","trace[2018551126] 'range keys from in-memory index tree' (duration: 386.372712ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:37.622311Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"386.423523ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-17T20:06:37.622880Z","caller":"traceutil/trace.go:172","msg":"trace[680880734] linearizableReadLoop","detail":"{readStateIndex:792; appliedIndex:791; }","duration":"387.273526ms","start":"2026-03-17T20:06:37.235595Z","end":"2026-03-17T20:06:37.622869Z","steps":["trace[680880734] 'read index received' (duration: 973.686µs)","trace[680880734] 'applied index is now lower than readState.Index' (duration: 386.29819ms)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:06:37.622913Z","caller":"traceutil/trace.go:172","msg":"trace[63606521] transaction","detail":"{read_only:false; response_revision:733; number_of_response:1; }","duration":"840.95933ms","start":"2026-03-17T20:06:36.781933Z","end":"2026-03-17T20:06:37.622892Z","steps":["trace[63606521] 'process raft request' (duration: 453.853507ms)","trace[63606521] 'compare' (duration: 386.24708ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:37.623053Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:36.781906Z","time spent":"841.061972ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":468,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:37.623115Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"995.543757ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:37.623172Z","caller":"traceutil/trace.go:172","msg":"trace[1545483377] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:734; }","duration":"995.600578ms","start":"2026-03-17T20:06:36.627564Z","end":"2026-03-17T20:06:37.623165Z","steps":["trace[1545483377] 'agreement among raft nodes before linearized reading' (duration: 995.516777ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:37.623197Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:36.627540Z","time spent":"995.651879ms","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:06:38.059777Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"259.85098ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-17T20:06:38.123343Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931979905,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:06:38.131552Z","caller":"traceutil/trace.go:172","msg":"trace[990884663] linearizableReadLoop","detail":"{readStateIndex:793; appliedIndex:794; }","duration":"508.595993ms","start":"2026-03-17T20:06:37.622934Z","end":"2026-03-17T20:06:38.131530Z","steps":["trace[990884663] 'read index received' (duration: 508.589243ms)","trace[990884663] 'applied index is now lower than readState.Index' (duration: 5.61µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:38.132138Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"511.335415ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-17T20:06:38.132219Z","caller":"traceutil/trace.go:172","msg":"trace[182173061] transaction","detail":"{read_only:false; response_revision:736; number_of_response:1; }","duration":"677.675743ms","start":"2026-03-17T20:06:37.454516Z","end":"2026-03-17T20:06:38.132192Z","steps":["trace[182173061] 'process raft request' (duration: 677.531321ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:38.132253Z","caller":"traceutil/trace.go:172","msg":"trace[638995995] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:735; }","duration":"511.478667ms","start":"2026-03-17T20:06:37.620757Z","end":"2026-03-17T20:06:38.132236Z","steps":["trace[638995995] 'agreement among raft nodes before linearized reading' (duration: 511.121891ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:38.132373Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:37.454472Z","time spent":"677.816494ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":494,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:38.132306Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:37.620741Z","time spent":"511.547278ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:38.132360Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"510.721425ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-17T20:06:38.132526Z","caller":"traceutil/trace.go:172","msg":"trace[48490204] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:735; }","duration":"510.913247ms","start":"2026-03-17T20:06:37.621592Z","end":"2026-03-17T20:06:38.132505Z","steps":["trace[48490204] 'agreement among raft nodes before linearized reading' (duration: 510.485951ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:38.132586Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:37.621575Z","time spent":"511.000018ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:38.133305Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"509.087652ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:06:38.133353Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"509.079682ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:38.133365Z","caller":"traceutil/trace.go:172","msg":"trace[829653743] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:736; }","duration":"509.143303ms","start":"2026-03-17T20:06:37.624204Z","end":"2026-03-17T20:06:38.133347Z","steps":["trace[829653743] 'agreement among raft nodes before linearized reading' (duration: 509.063391ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:38.134599Z","caller":"traceutil/trace.go:172","msg":"trace[405493360] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:736; }","duration":"509.121892ms","start":"2026-03-17T20:06:37.624260Z","end":"2026-03-17T20:06:38.133382Z","steps":["trace[405493360] 'agreement among raft nodes before linearized reading' (duration: 509.054271ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:38.134839Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:37.624244Z","time spent":"510.553103ms","remote":"127.0.0.1:46490","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:06:38.608837Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"226.775444ms"} {"level":"warn","ts":"2026-03-17T20:06:38.608933Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"226.879185ms"} {"level":"info","ts":"2026-03-17T20:06:38.609053Z","caller":"traceutil/trace.go:172","msg":"trace[432845702] linearizableReadLoop","detail":"{readStateIndex:795; appliedIndex:795; }","duration":"456.102052ms","start":"2026-03-17T20:06:38.152926Z","end":"2026-03-17T20:06:38.609028Z","steps":["trace[432845702] 'read index received' (duration: 456.092961ms)","trace[432845702] 'applied index is now lower than readState.Index' (duration: 7.211µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:38.609195Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"456.242443ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:38.609235Z","caller":"traceutil/trace.go:172","msg":"trace[70015220] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:736; }","duration":"456.307164ms","start":"2026-03-17T20:06:38.152918Z","end":"2026-03-17T20:06:38.609225Z","steps":["trace[70015220] 'agreement among raft nodes before linearized reading' (duration: 456.184573ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:38.609283Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:38.152886Z","time spent":"456.386887ms","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-17T20:06:38.609564Z","caller":"traceutil/trace.go:172","msg":"trace[1357882683] transaction","detail":"{read_only:false; response_revision:738; number_of_response:1; }","duration":"469.92849ms","start":"2026-03-17T20:06:38.139620Z","end":"2026-03-17T20:06:38.609549Z","steps":["trace[1357882683] 'process raft request' (duration: 469.875059ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:38.609723Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:38.139595Z","time spent":"470.031001ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":427,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:38.609872Z","caller":"traceutil/trace.go:172","msg":"trace[1359764862] transaction","detail":"{read_only:false; response_revision:737; number_of_response:1; }","duration":"472.197543ms","start":"2026-03-17T20:06:38.137588Z","end":"2026-03-17T20:06:38.609785Z","steps":["trace[1359764862] 'process raft request' (duration: 471.765467ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:38.610233Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:38.137561Z","time spent":"472.489977ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":423,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:38.824119Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"12.416971ms"} {"level":"warn","ts":"2026-03-17T20:06:38.824204Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"12.511692ms"} {"level":"info","ts":"2026-03-17T20:06:38.881507Z","caller":"traceutil/trace.go:172","msg":"trace[1477645859] linearizableReadLoop","detail":"{readStateIndex:797; appliedIndex:799; }","duration":"256.823837ms","start":"2026-03-17T20:06:38.624643Z","end":"2026-03-17T20:06:38.881467Z","steps":["trace[1477645859] 'read index received' (duration: 256.806817ms)","trace[1477645859] 'applied index is now lower than readState.Index' (duration: 15µs)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:06:38.881635Z","caller":"traceutil/trace.go:172","msg":"trace[133013495] transaction","detail":"{read_only:false; response_revision:740; number_of_response:1; }","duration":"167.806703ms","start":"2026-03-17T20:06:38.713796Z","end":"2026-03-17T20:06:38.881603Z","steps":["trace[133013495] 'process raft request' (duration: 167.632011ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:38.881697Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"256.981699ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:38.881981Z","caller":"traceutil/trace.go:172","msg":"trace[1831338742] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:740; }","duration":"257.330005ms","start":"2026-03-17T20:06:38.624633Z","end":"2026-03-17T20:06:38.881963Z","steps":["trace[1831338742] 'agreement among raft nodes before linearized reading' (duration: 256.938319ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:39.672401Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"154.373016ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:39.672763Z","caller":"traceutil/trace.go:172","msg":"trace[1890211793] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:742; }","duration":"154.763271ms","start":"2026-03-17T20:06:39.517985Z","end":"2026-03-17T20:06:39.672748Z","steps":["trace[1890211793] 'agreement among raft nodes before linearized reading' (duration: 87.719511ms)","trace[1890211793] 'range keys from in-memory index tree' (duration: 66.601404ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:40.905417Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"89.580725ms"} {"level":"warn","ts":"2026-03-17T20:06:40.905487Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"89.659646ms"} {"level":"info","ts":"2026-03-17T20:06:40.905908Z","caller":"traceutil/trace.go:172","msg":"trace[378440869] transaction","detail":"{read_only:false; response_revision:745; number_of_response:1; }","duration":"285.965162ms","start":"2026-03-17T20:06:40.619896Z","end":"2026-03-17T20:06:40.905862Z","steps":["trace[378440869] 'process raft request' (duration: 285.700857ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:40.908353Z","caller":"traceutil/trace.go:172","msg":"trace[1125464169] transaction","detail":"{read_only:false; response_revision:746; number_of_response:1; }","duration":"287.477173ms","start":"2026-03-17T20:06:40.620862Z","end":"2026-03-17T20:06:40.908340Z","steps":["trace[1125464169] 'process raft request' (duration: 287.094928ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:40.908488Z","caller":"etcdserver/server.go:2246","msg":"skip compaction since there is an inflight snapshot"} {"level":"warn","ts":"2026-03-17T20:06:47.285953Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"124.07664ms"} {"level":"warn","ts":"2026-03-17T20:06:47.286038Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"124.175221ms"} {"level":"info","ts":"2026-03-17T20:06:47.286486Z","caller":"traceutil/trace.go:172","msg":"trace[643954915] transaction","detail":"{read_only:false; response_revision:766; number_of_response:1; }","duration":"273.062902ms","start":"2026-03-17T20:06:47.013397Z","end":"2026-03-17T20:06:47.286459Z","steps":["trace[643954915] 'process raft request' (duration: 272.95252ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:47.286582Z","caller":"traceutil/trace.go:172","msg":"trace[1445106539] transaction","detail":"{read_only:false; response_revision:765; number_of_response:1; }","duration":"273.849473ms","start":"2026-03-17T20:06:47.012695Z","end":"2026-03-17T20:06:47.286545Z","steps":["trace[1445106539] 'process raft request' (duration: 273.409026ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:47.446079Z","caller":"traceutil/trace.go:172","msg":"trace[678041767] transaction","detail":"{read_only:false; response_revision:767; number_of_response:1; }","duration":"143.251719ms","start":"2026-03-17T20:06:47.302781Z","end":"2026-03-17T20:06:47.446033Z","steps":["trace[678041767] 'process raft request' (duration: 142.938704ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:47.724487Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"77.886441ms"} {"level":"warn","ts":"2026-03-17T20:06:47.724558Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"77.971143ms"} {"level":"warn","ts":"2026-03-17T20:06:47.963267Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"445.741122ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:47.963341Z","caller":"traceutil/trace.go:172","msg":"trace[1246229833] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:767; }","duration":"445.825493ms","start":"2026-03-17T20:06:47.517503Z","end":"2026-03-17T20:06:47.963328Z","steps":["trace[1246229833] 'range keys from in-memory index tree' (duration: 445.687832ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:47.963622Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"334.629884ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:47.963647Z","caller":"traceutil/trace.go:172","msg":"trace[14761330] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:767; }","duration":"334.654794ms","start":"2026-03-17T20:06:47.628984Z","end":"2026-03-17T20:06:47.963639Z","steps":["trace[14761330] 'agreement among raft nodes before linearized reading' (duration: 96.691427ms)","trace[14761330] 'range keys from in-memory index tree' (duration: 237.919467ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:47.963669Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:47.628948Z","time spent":"334.716245ms","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:06:47.964061Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"239.334038ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-17T20:06:47.964273Z","caller":"traceutil/trace.go:172","msg":"trace[1777939996] transaction","detail":"{read_only:false; response_revision:769; number_of_response:1; }","duration":"165.270312ms","start":"2026-03-17T20:06:47.798989Z","end":"2026-03-17T20:06:47.964260Z","steps":["trace[1777939996] 'process raft request' (duration: 165.172821ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:47.964370Z","caller":"traceutil/trace.go:172","msg":"trace[377239357] transaction","detail":"{read_only:false; response_revision:768; number_of_response:1; }","duration":"506.794426ms","start":"2026-03-17T20:06:47.457548Z","end":"2026-03-17T20:06:47.964343Z","steps":["trace[377239357] 'process raft request' (duration: 267.122944ms)","trace[377239357] 'compare' (duration: 238.83546ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:47.964516Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:47.457513Z","time spent":"506.945139ms","remote":"127.0.0.1:47520","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":3960,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-17T20:06:48.188048Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"23.176635ms"} {"level":"warn","ts":"2026-03-17T20:06:48.188199Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"23.328717ms"} {"level":"warn","ts":"2026-03-17T20:06:48.643038Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"125.350534ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:48.643100Z","caller":"traceutil/trace.go:172","msg":"trace[1444651728] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:769; }","duration":"125.420065ms","start":"2026-03-17T20:06:48.517669Z","end":"2026-03-17T20:06:48.643089Z","steps":["trace[1444651728] 'range keys from in-memory index tree' (duration: 125.299692ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:48.643326Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"455.088954ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-17T20:06:48.643495Z","caller":"traceutil/trace.go:172","msg":"trace[327628271] transaction","detail":"{read_only:false; response_revision:770; number_of_response:1; }","duration":"673.014531ms","start":"2026-03-17T20:06:47.970466Z","end":"2026-03-17T20:06:48.643481Z","steps":["trace[327628271] 'process raft request' (duration: 217.666514ms)","trace[327628271] 'compare' (duration: 454.974632ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:48.643646Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:47.970436Z","time spent":"673.189413ms","remote":"127.0.0.1:46640","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":687,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-17T20:06:48.652962Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"264.727175ms"} {"level":"warn","ts":"2026-03-17T20:06:48.653034Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"264.812146ms"} {"level":"info","ts":"2026-03-17T20:06:48.653483Z","caller":"traceutil/trace.go:172","msg":"trace[37459268] transaction","detail":"{read_only:false; response_revision:771; number_of_response:1; }","duration":"679.351893ms","start":"2026-03-17T20:06:47.974108Z","end":"2026-03-17T20:06:48.653459Z","steps":["trace[37459268] 'process raft request' (duration: 679.16069ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:48.653537Z","caller":"traceutil/trace.go:172","msg":"trace[231338088] transaction","detail":"{read_only:false; response_revision:772; number_of_response:1; }","duration":"679.056369ms","start":"2026-03-17T20:06:47.974453Z","end":"2026-03-17T20:06:48.653509Z","steps":["trace[231338088] 'process raft request' (duration: 678.900366ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:48.653722Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:47.974445Z","time spent":"679.197501ms","remote":"127.0.0.1:47484","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4462,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:48.654010Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:47.974093Z","time spent":"679.446566ms","remote":"127.0.0.1:46858","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":3848,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-17T20:06:48.807319Z","caller":"traceutil/trace.go:172","msg":"trace[246434631] linearizableReadLoop","detail":"{readStateIndex:837; appliedIndex:838; }","duration":"164.15469ms","start":"2026-03-17T20:06:48.643132Z","end":"2026-03-17T20:06:48.807287Z","steps":["trace[246434631] 'read index received' (duration: 164.14642ms)","trace[246434631] 'applied index is now lower than readState.Index' (duration: 7.11µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:49.076725Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"433.555797ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:49.076819Z","caller":"traceutil/trace.go:172","msg":"trace[440923017] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:773; }","duration":"433.674349ms","start":"2026-03-17T20:06:48.643125Z","end":"2026-03-17T20:06:49.076800Z","steps":["trace[440923017] 'agreement among raft nodes before linearized reading' (duration: 164.282752ms)","trace[440923017] 'range keys from in-memory index tree' (duration: 269.232605ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:49.077124Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"269.772102ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-17T20:06:49.077326Z","caller":"traceutil/trace.go:172","msg":"trace[2023287308] transaction","detail":"{read_only:false; response_revision:774; number_of_response:1; }","duration":"812.991794ms","start":"2026-03-17T20:06:48.264320Z","end":"2026-03-17T20:06:49.077311Z","steps":["trace[2023287308] 'process raft request' (duration: 542.945168ms)","trace[2023287308] 'compare' (duration: 269.62359ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:49.077420Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:48.264293Z","time spent":"813.062535ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":493,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:49.133854Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"280.793197ms"} {"level":"warn","ts":"2026-03-17T20:06:49.133923Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"280.871348ms"} {"level":"info","ts":"2026-03-17T20:06:49.134298Z","caller":"traceutil/trace.go:172","msg":"trace[1807114131] transaction","detail":"{read_only:false; response_revision:776; number_of_response:1; }","duration":"474.333521ms","start":"2026-03-17T20:06:48.659936Z","end":"2026-03-17T20:06:49.134270Z","steps":["trace[1807114131] 'process raft request' (duration: 474.032407ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:49.134346Z","caller":"traceutil/trace.go:172","msg":"trace[534861481] transaction","detail":"{read_only:false; response_revision:777; number_of_response:1; }","duration":"474.308571ms","start":"2026-03-17T20:06:48.660002Z","end":"2026-03-17T20:06:49.134310Z","steps":["trace[534861481] 'process raft request' (duration: 474.195589ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:49.134452Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:48.659914Z","time spent":"474.472863ms","remote":"127.0.0.1:46640","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":704,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-17T20:06:49.134501Z","caller":"traceutil/trace.go:172","msg":"trace[1968811] transaction","detail":"{read_only:false; response_revision:778; number_of_response:1; }","duration":"473.410749ms","start":"2026-03-17T20:06:48.661067Z","end":"2026-03-17T20:06:49.134478Z","steps":["trace[1968811] 'process raft request' (duration: 473.154154ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:49.134885Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:48.661043Z","time spent":"473.625381ms","remote":"127.0.0.1:47520","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4086,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:49.134900Z","caller":"traceutil/trace.go:172","msg":"trace[734355202] transaction","detail":"{read_only:false; response_revision:780; number_of_response:1; }","duration":"471.939356ms","start":"2026-03-17T20:06:48.662709Z","end":"2026-03-17T20:06:49.134648Z","steps":["trace[734355202] 'process raft request' (duration: 471.905876ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:49.135617Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:48.662645Z","time spent":"472.88507ms","remote":"127.0.0.1:46858","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4378,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:49.137559Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:48.659979Z","time spent":"474.430142ms","remote":"127.0.0.1:46640","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":858,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-17T20:06:49.134736Z","caller":"traceutil/trace.go:172","msg":"trace[1030925386] transaction","detail":"{read_only:false; response_revision:779; number_of_response:1; }","duration":"473.521679ms","start":"2026-03-17T20:06:48.661205Z","end":"2026-03-17T20:06:49.134727Z","steps":["trace[1030925386] 'process raft request' (duration: 473.103243ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:49.153356Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:48.661185Z","time spent":"492.07912ms","remote":"127.0.0.1:47520","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4077,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:49.418387Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"37.689127ms"} {"level":"warn","ts":"2026-03-17T20:06:49.418602Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"37.91263ms"} {"level":"info","ts":"2026-03-17T20:06:49.418709Z","caller":"traceutil/trace.go:172","msg":"trace[1926045588] transaction","detail":"{read_only:false; response_revision:781; number_of_response:1; }","duration":"466.403038ms","start":"2026-03-17T20:06:48.952290Z","end":"2026-03-17T20:06:49.418693Z","steps":["trace[1926045588] 'process raft request' (duration: 466.222864ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:49.418836Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:48.952271Z","time spent":"466.488088ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":537,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:49.441506Z","caller":"traceutil/trace.go:172","msg":"trace[1716775797] linearizableReadLoop","detail":"{readStateIndex:845; appliedIndex:846; }","duration":"303.58803ms","start":"2026-03-17T20:06:49.137895Z","end":"2026-03-17T20:06:49.441483Z","steps":["trace[1716775797] 'read index received' (duration: 303.579899ms)","trace[1716775797] 'applied index is now lower than readState.Index' (duration: 6.411µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:49.784851Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"5.006021ms"} {"level":"warn","ts":"2026-03-17T20:06:49.784923Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"5.086712ms"} {"level":"warn","ts":"2026-03-17T20:06:49.811490Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"673.566687ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/pods/kube-system/coredns-66bc5c9577-cbsms\" limit:1 ","response":"range_response_count:1 size:4357"} {"level":"info","ts":"2026-03-17T20:06:49.811579Z","caller":"traceutil/trace.go:172","msg":"trace[1182789211] range","detail":"{range_begin:/registry/pods/kube-system/coredns-66bc5c9577-cbsms; range_end:; response_count:1; response_revision:781; }","duration":"673.668218ms","start":"2026-03-17T20:06:49.137891Z","end":"2026-03-17T20:06:49.811559Z","steps":["trace[1182789211] 'agreement among raft nodes before linearized reading' (duration: 303.801502ms)","trace[1182789211] 'range keys from in-memory index tree' (duration: 369.627572ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:49.811559Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"369.734975ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-17T20:06:49.811612Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.137874Z","time spent":"673.73044ms","remote":"127.0.0.1:46858","response type":"/etcdserverpb.KV/Range","request count":0,"request size":55,"response count":1,"response size":4380,"request content":"key:\"/registry/pods/kube-system/coredns-66bc5c9577-cbsms\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:49.811504Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"293.647604ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:49.811660Z","caller":"traceutil/trace.go:172","msg":"trace[218090672] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:781; }","duration":"293.821727ms","start":"2026-03-17T20:06:49.517823Z","end":"2026-03-17T20:06:49.811645Z","steps":["trace[218090672] 'range keys from in-memory index tree' (duration: 293.588133ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:49.811718Z","caller":"traceutil/trace.go:172","msg":"trace[2008951344] transaction","detail":"{read_only:false; response_revision:782; number_of_response:1; }","duration":"671.00816ms","start":"2026-03-17T20:06:49.140701Z","end":"2026-03-17T20:06:49.811709Z","steps":["trace[2008951344] 'process raft request' (duration: 301.082354ms)","trace[2008951344] 'compare' (duration: 369.635572ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:49.811785Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.140679Z","time spent":"671.059801ms","remote":"127.0.0.1:46640","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":689,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-17T20:06:49.812208Z","caller":"traceutil/trace.go:172","msg":"trace[1574607599] transaction","detail":"{read_only:false; response_revision:783; number_of_response:1; }","duration":"653.043577ms","start":"2026-03-17T20:06:49.159107Z","end":"2026-03-17T20:06:49.812150Z","steps":["trace[1574607599] 'process raft request' (duration: 652.520149ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:49.812489Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.159075Z","time spent":"653.349071ms","remote":"127.0.0.1:47520","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4122,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:49.812268Z","caller":"traceutil/trace.go:172","msg":"trace[1847387817] linearizableReadLoop","detail":"{readStateIndex:848; appliedIndex:846; }","duration":"370.687858ms","start":"2026-03-17T20:06:49.441569Z","end":"2026-03-17T20:06:49.812257Z","steps":["trace[1847387817] 'read index received' (duration: 1.070666ms)","trace[1847387817] 'applied index is now lower than readState.Index' (duration: 369.615412ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:49.845621Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"552.58011ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-17T20:06:49.845714Z","caller":"traceutil/trace.go:172","msg":"trace[1566078400] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:783; }","duration":"552.742753ms","start":"2026-03-17T20:06:49.292949Z","end":"2026-03-17T20:06:49.845692Z","steps":["trace[1566078400] 'agreement among raft nodes before linearized reading' (duration: 519.240545ms)","trace[1566078400] 'range keys from in-memory index tree' (duration: 32.672196ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:49.845656Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"705.274419ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/deployments/kube-system/coredns\" limit:1 ","response":"range_response_count:1 size:4477"} {"level":"info","ts":"2026-03-17T20:06:49.845902Z","caller":"traceutil/trace.go:172","msg":"trace[1030904751] range","detail":"{range_begin:/registry/deployments/kube-system/coredns; range_end:; response_count:1; response_revision:783; }","duration":"705.533483ms","start":"2026-03-17T20:06:49.140344Z","end":"2026-03-17T20:06:49.845877Z","steps":["trace[1030904751] 'agreement among raft nodes before linearized reading' (duration: 671.944823ms)","trace[1030904751] 'range keys from in-memory index tree' (duration: 32.505174ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:49.845961Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.140314Z","time spent":"705.626025ms","remote":"127.0.0.1:47484","response type":"/etcdserverpb.KV/Range","request count":0,"request size":45,"response count":1,"response size":4500,"request content":"key:\"/registry/deployments/kube-system/coredns\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:49.846421Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.292925Z","time spent":"553.475144ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:49.847116Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"553.828429ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-17T20:06:49.847162Z","caller":"traceutil/trace.go:172","msg":"trace[1204001643] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:783; }","duration":"553.872821ms","start":"2026-03-17T20:06:49.293276Z","end":"2026-03-17T20:06:49.847148Z","steps":["trace[1204001643] 'agreement among raft nodes before linearized reading' (duration: 518.896081ms)","trace[1204001643] 'range keys from in-memory index tree' (duration: 32.640165ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:49.847189Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.293251Z","time spent":"553.930791ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:49.848281Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"519.283046ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true ","response":"range_response_count:61 size:5153"} {"level":"info","ts":"2026-03-17T20:06:49.848445Z","caller":"traceutil/trace.go:172","msg":"trace[1851544848] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; response_count:61; response_revision:783; }","duration":"519.485248ms","start":"2026-03-17T20:06:49.328944Z","end":"2026-03-17T20:06:49.848429Z","steps":["trace[1851544848] 'agreement among raft nodes before linearized reading' (duration: 483.238861ms)","trace[1851544848] 'range keys from in-memory index tree' (duration: 34.669855ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:49.848596Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.328921Z","time spent":"519.659901ms","remote":"127.0.0.1:46640","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":61,"response size":5176,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true "} {"level":"warn","ts":"2026-03-17T20:06:50.044870Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"200.170249ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-17T20:06:50.044995Z","caller":"traceutil/trace.go:172","msg":"trace[1113804914] linearizableReadLoop","detail":"{readStateIndex:850; appliedIndex:849; }","duration":"232.311368ms","start":"2026-03-17T20:06:49.812665Z","end":"2026-03-17T20:06:50.044976Z","steps":["trace[1113804914] 'read index received' (duration: 33.054842ms)","trace[1113804914] 'applied index is now lower than readState.Index' (duration: 199.255056ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:50.045072Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"233.349603ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:06:50.045112Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"416.610566ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:50.045154Z","caller":"traceutil/trace.go:172","msg":"trace[1023688506] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:783; }","duration":"233.447864ms","start":"2026-03-17T20:06:49.811689Z","end":"2026-03-17T20:06:50.045137Z","steps":["trace[1023688506] 'agreement among raft nodes before linearized reading' (duration: 233.316202ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:50.045164Z","caller":"traceutil/trace.go:172","msg":"trace[823076173] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:783; }","duration":"416.660007ms","start":"2026-03-17T20:06:49.628488Z","end":"2026-03-17T20:06:50.045148Z","steps":["trace[823076173] 'agreement among raft nodes before linearized reading' (duration: 416.581906ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:50.045217Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.628460Z","time spent":"416.745138ms","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:06:50.480689Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"434.122524ms"} {"level":"warn","ts":"2026-03-17T20:06:50.480774Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"434.216876ms"} {"level":"info","ts":"2026-03-17T20:06:50.481044Z","caller":"traceutil/trace.go:172","msg":"trace[105613377] transaction","detail":"{read_only:false; response_revision:784; number_of_response:1; }","duration":"668.06559ms","start":"2026-03-17T20:06:49.812960Z","end":"2026-03-17T20:06:50.481026Z","steps":["trace[105613377] 'process raft request' (duration: 667.873547ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:50.481169Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.812942Z","time spent":"668.162991ms","remote":"127.0.0.1:46858","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4354,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:50.481551Z","caller":"traceutil/trace.go:172","msg":"trace[1681927506] transaction","detail":"{read_only:false; response_revision:787; number_of_response:1; }","duration":"509.902334ms","start":"2026-03-17T20:06:49.971591Z","end":"2026-03-17T20:06:50.481493Z","steps":["trace[1681927506] 'process raft request' (duration: 509.838533ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:50.481593Z","caller":"traceutil/trace.go:172","msg":"trace[2066858678] transaction","detail":"{read_only:false; response_revision:785; number_of_response:1; }","duration":"629.787892ms","start":"2026-03-17T20:06:49.851796Z","end":"2026-03-17T20:06:50.481584Z","steps":["trace[2066858678] 'process raft request' (duration: 629.185403ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:50.481637Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.851780Z","time spent":"629.835412ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":423,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:50.481673Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.971561Z","time spent":"510.037455ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":468,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:50.481787Z","caller":"traceutil/trace.go:172","msg":"trace[395258718] transaction","detail":"{read_only:false; response_revision:786; number_of_response:1; }","duration":"629.346626ms","start":"2026-03-17T20:06:49.852425Z","end":"2026-03-17T20:06:50.481771Z","steps":["trace[395258718] 'process raft request' (duration: 628.841458ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:50.481920Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.852415Z","time spent":"629.466897ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":427,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:50.507505Z","caller":"traceutil/trace.go:172","msg":"trace[767998697] linearizableReadLoop","detail":"{readStateIndex:850; appliedIndex:854; }","duration":"462.459002ms","start":"2026-03-17T20:06:50.045021Z","end":"2026-03-17T20:06:50.507480Z","steps":["trace[767998697] 'read index received' (duration: 462.449892ms)","trace[767998697] 'applied index is now lower than readState.Index' (duration: 7.45µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:50.507902Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"655.403604ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/replicasets/kube-system/coredns-7488bc8bd5\" limit:1 ","response":"range_response_count:1 size:4137"} {"level":"info","ts":"2026-03-17T20:06:50.508203Z","caller":"traceutil/trace.go:172","msg":"trace[1836481331] range","detail":"{range_begin:/registry/replicasets/kube-system/coredns-7488bc8bd5; range_end:; response_count:1; response_revision:787; }","duration":"655.617358ms","start":"2026-03-17T20:06:49.852406Z","end":"2026-03-17T20:06:50.508024Z","steps":["trace[1836481331] 'agreement among raft nodes before linearized reading' (duration: 655.202562ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:50.508267Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:49.852369Z","time spent":"655.887331ms","remote":"127.0.0.1:47520","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":1,"response size":4160,"request content":"key:\"/registry/replicasets/kube-system/coredns-7488bc8bd5\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:50.834490Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"91.132377ms"} {"level":"warn","ts":"2026-03-17T20:06:50.834573Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"91.222619ms"} {"level":"warn","ts":"2026-03-17T20:06:51.008952Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980014,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:06:51.167321Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"120.345336ms"} {"level":"warn","ts":"2026-03-17T20:06:51.167378Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"120.410307ms"} {"level":"info","ts":"2026-03-17T20:06:51.167347Z","caller":"traceutil/trace.go:172","msg":"trace[405518932] linearizableReadLoop","detail":"{readStateIndex:854; appliedIndex:854; }","duration":"659.346511ms","start":"2026-03-17T20:06:50.507981Z","end":"2026-03-17T20:06:51.167328Z","steps":["trace[405518932] 'read index received' (duration: 659.338631ms)","trace[405518932] 'applied index is now lower than readState.Index' (duration: 6.78µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:51.224361Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.17737947s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:51.224431Z","caller":"traceutil/trace.go:172","msg":"trace[1404293302] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:787; }","duration":"1.177457981s","start":"2026-03-17T20:06:50.046962Z","end":"2026-03-17T20:06:51.224420Z","steps":["trace[1404293302] 'agreement among raft nodes before linearized reading' (duration: 1.120413052s)","trace[1404293302] 'range keys from in-memory index tree' (duration: 56.924447ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:51.224636Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:50.046943Z","time spent":"1.177687304s","remote":"127.0.0.1:46490","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:06:51.224460Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"304.015647ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:51.224731Z","caller":"traceutil/trace.go:172","msg":"trace[1418742936] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:787; }","duration":"304.294261ms","start":"2026-03-17T20:06:50.920432Z","end":"2026-03-17T20:06:51.224726Z","steps":["trace[1418742936] 'range keys from in-memory index tree' (duration: 303.975237ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:51.224698Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"389.887958ms","expected-duration":"100ms","prefix":"","request":"header: txn: success: > failure: >>","response":"size:4380"} {"level":"info","ts":"2026-03-17T20:06:51.224914Z","caller":"traceutil/trace.go:172","msg":"trace[589943473] transaction","detail":"{read_only:false; number_of_response:1; response_revision:787; }","duration":"716.955478ms","start":"2026-03-17T20:06:50.507933Z","end":"2026-03-17T20:06:51.224888Z","steps":["trace[589943473] 'process raft request' (duration: 326.704865ms)","trace[589943473] 'compare' (duration: 389.802897ms)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:06:51.225417Z","caller":"traceutil/trace.go:172","msg":"trace[873229785] transaction","detail":"{read_only:false; response_revision:788; number_of_response:1; }","duration":"707.267027ms","start":"2026-03-17T20:06:50.518142Z","end":"2026-03-17T20:06:51.225409Z","steps":["trace[873229785] 'process raft request' (duration: 706.660158ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:51.225397Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:50.507910Z","time spent":"717.361224ms","remote":"127.0.0.1:46858","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":55,"response count":0,"response size":4403,"request content":"compare: success: > failure: >"} {"level":"warn","ts":"2026-03-17T20:06:51.225479Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:50.518113Z","time spent":"707.331827ms","remote":"127.0.0.1:47520","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4122,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:51.257963Z","caller":"etcdserver/server.go:1856","msg":"sent merged snapshot","from":"c1a97aa766b87c7e","to":"c6d6ded81dced554","bytes":1032751,"size":"1.0 MB","took":"30.016326159s"} {"level":"warn","ts":"2026-03-17T20:06:51.261523Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":8970780119931980017,"received-request-id":8970780119931980014} {"level":"info","ts":"2026-03-17T20:06:51.261743Z","caller":"traceutil/trace.go:172","msg":"trace[1352115612] transaction","detail":"{read_only:false; response_revision:789; number_of_response:1; }","duration":"179.202132ms","start":"2026-03-17T20:06:51.082521Z","end":"2026-03-17T20:06:51.261723Z","steps":["trace[1352115612] 'process raft request' (duration: 179.068461ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:51.575751Z","caller":"traceutil/trace.go:172","msg":"trace[1021473629] linearizableReadLoop","detail":"{readStateIndex:857; appliedIndex:858; }","duration":"408.22917ms","start":"2026-03-17T20:06:51.167494Z","end":"2026-03-17T20:06:51.575723Z","steps":["trace[1021473629] 'read index received' (duration: 408.22172ms)","trace[1021473629] 'applied index is now lower than readState.Index' (duration: 6.04µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:51.575860Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"114.484717ms"} {"level":"warn","ts":"2026-03-17T20:06:51.575910Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"114.523437ms"} {"level":"warn","ts":"2026-03-17T20:06:51.926490Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.408078645s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:51.926639Z","caller":"traceutil/trace.go:172","msg":"trace[661067986] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:789; }","duration":"1.408241787s","start":"2026-03-17T20:06:50.518351Z","end":"2026-03-17T20:06:51.926593Z","steps":["trace[661067986] 'agreement among raft nodes before linearized reading' (duration: 1.057505444s)","trace[661067986] 'range keys from in-memory index tree' (duration: 350.532531ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:51.927161Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"351.138149ms","expected-duration":"100ms","prefix":"","request":"header: txn: success: > failure: >>","response":"size:18"} {"level":"info","ts":"2026-03-17T20:06:51.927431Z","caller":"traceutil/trace.go:172","msg":"trace[2145779771] transaction","detail":"{read_only:false; response_revision:791; number_of_response:1; }","duration":"697.881855ms","start":"2026-03-17T20:06:51.229527Z","end":"2026-03-17T20:06:51.927409Z","steps":["trace[2145779771] 'process raft request' (duration: 697.758103ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:51.927526Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:51.229508Z","time spent":"697.975407ms","remote":"127.0.0.1:46640","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":687,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-17T20:06:51.927677Z","caller":"traceutil/trace.go:172","msg":"trace[411913317] transaction","detail":"{read_only:false; number_of_response:1; response_revision:790; }","duration":"700.152208ms","start":"2026-03-17T20:06:51.227506Z","end":"2026-03-17T20:06:51.927658Z","steps":["trace[411913317] 'process raft request' (duration: 348.39156ms)","trace[411913317] 'compare' (duration: 350.733094ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:51.929073Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:51.227481Z","time spent":"700.28995ms","remote":"127.0.0.1:46858","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":55,"response count":0,"response size":41,"request content":"compare: success: > failure: >"} {"level":"warn","ts":"2026-03-17T20:06:52.076643Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980022,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:06:52.143514Z","caller":"traceutil/trace.go:172","msg":"trace[1316865109] linearizableReadLoop","detail":"{readStateIndex:860; appliedIndex:860; }","duration":"567.66756ms","start":"2026-03-17T20:06:51.575825Z","end":"2026-03-17T20:06:52.143493Z","steps":["trace[1316865109] 'read index received' (duration: 567.65633ms)","trace[1316865109] 'applied index is now lower than readState.Index' (duration: 10.19µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:52.143694Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"96.639994ms"} {"level":"warn","ts":"2026-03-17T20:06:52.143734Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"96.681035ms"} {"level":"warn","ts":"2026-03-17T20:06:52.246205Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.013853527s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:52.246324Z","caller":"traceutil/trace.go:172","msg":"trace[752786424] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:791; }","duration":"1.013996839s","start":"2026-03-17T20:06:51.232300Z","end":"2026-03-17T20:06:52.246297Z","steps":["trace[752786424] 'agreement among raft nodes before linearized reading' (duration: 911.296731ms)","trace[752786424] 'range keys from in-memory index tree' (duration: 102.514595ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:52.246315Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"102.384574ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-17T20:06:52.246383Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:51.232291Z","time spent":"1.014079611s","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-17T20:06:52.246579Z","caller":"traceutil/trace.go:172","msg":"trace[1958755828] transaction","detail":"{read_only:false; response_revision:792; number_of_response:1; }","duration":"1.016122881s","start":"2026-03-17T20:06:51.230423Z","end":"2026-03-17T20:06:52.246546Z","steps":["trace[1958755828] 'process raft request' (duration: 913.491033ms)","trace[1958755828] 'compare' (duration: 102.133541ms)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:06:52.246663Z","caller":"traceutil/trace.go:172","msg":"trace[616544822] transaction","detail":"{read_only:false; response_revision:795; number_of_response:1; }","duration":"313.050929ms","start":"2026-03-17T20:06:51.933595Z","end":"2026-03-17T20:06:52.246646Z","steps":["trace[616544822] 'process raft request' (duration: 313.001369ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:52.246744Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:51.933562Z","time spent":"313.14434ms","remote":"127.0.0.1:46640","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":704,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"info","ts":"2026-03-17T20:06:52.247020Z","caller":"traceutil/trace.go:172","msg":"trace[802862919] transaction","detail":"{read_only:false; response_revision:793; number_of_response:1; }","duration":"1.014640838s","start":"2026-03-17T20:06:51.232358Z","end":"2026-03-17T20:06:52.246999Z","steps":["trace[802862919] 'process raft request' (duration: 1.014083321s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:52.247547Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":8970780119931980025,"received-request-id":8970780119931980022} {"level":"warn","ts":"2026-03-17T20:06:52.248710Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:51.232344Z","time spent":"1.016276552s","remote":"127.0.0.1:47484","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4462,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:52.248760Z","caller":"traceutil/trace.go:172","msg":"trace[804383203] linearizableReadLoop","detail":"{readStateIndex:864; appliedIndex:865; }","duration":"105.183965ms","start":"2026-03-17T20:06:52.143563Z","end":"2026-03-17T20:06:52.248747Z","steps":["trace[804383203] 'read index received' (duration: 105.176635ms)","trace[804383203] 'applied index is now lower than readState.Index' (duration: 6.37µs)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:06:52.247917Z","caller":"traceutil/trace.go:172","msg":"trace[2131534032] transaction","detail":"{read_only:false; response_revision:796; number_of_response:1; }","duration":"310.403932ms","start":"2026-03-17T20:06:51.937502Z","end":"2026-03-17T20:06:52.247906Z","steps":["trace[2131534032] 'process raft request' (duration: 310.27239ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:52.248836Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"318.389446ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:52.248860Z","caller":"traceutil/trace.go:172","msg":"trace[1348069508] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:796; }","duration":"318.429837ms","start":"2026-03-17T20:06:51.930422Z","end":"2026-03-17T20:06:52.248852Z","steps":["trace[1348069508] 'agreement among raft nodes before linearized reading' (duration: 318.363956ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:52.246681Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:51.230406Z","time spent":"1.016235972s","remote":"127.0.0.1:46858","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":3848,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-17T20:06:52.248913Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:51.937479Z","time spent":"311.356785ms","remote":"127.0.0.1:47520","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4086,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:52.248988Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"245.421607ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/specs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:52.249296Z","caller":"traceutil/trace.go:172","msg":"trace[142269238] range","detail":"{range_begin:/registry/services/specs; range_end:; response_count:0; response_revision:796; }","duration":"245.728922ms","start":"2026-03-17T20:06:52.003553Z","end":"2026-03-17T20:06:52.249282Z","steps":["trace[142269238] 'agreement among raft nodes before linearized reading' (duration: 245.279745ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:52.511071Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"101.333199ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"info","ts":"2026-03-17T20:06:52.511331Z","caller":"traceutil/trace.go:172","msg":"trace[594666528] transaction","detail":"{read_only:false; response_revision:797; number_of_response:1; }","duration":"257.625783ms","start":"2026-03-17T20:06:52.253646Z","end":"2026-03-17T20:06:52.511272Z","steps":["trace[594666528] 'process raft request' (duration: 155.540054ms)","trace[594666528] 'compare' (duration: 101.054454ms)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:06:52.531697Z","caller":"traceutil/trace.go:172","msg":"trace[1321545531] linearizableReadLoop","detail":"{readStateIndex:865; appliedIndex:866; }","duration":"272.307177ms","start":"2026-03-17T20:06:52.259365Z","end":"2026-03-17T20:06:52.531672Z","steps":["trace[1321545531] 'read index received' (duration: 272.297767ms)","trace[1321545531] 'applied index is now lower than readState.Index' (duration: 8.19µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:52.531838Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"272.457749ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:52.531883Z","caller":"traceutil/trace.go:172","msg":"trace[1163334237] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:797; }","duration":"272.506319ms","start":"2026-03-17T20:06:52.259361Z","end":"2026-03-17T20:06:52.531868Z","steps":["trace[1163334237] 'agreement among raft nodes before linearized reading' (duration: 272.427048ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:52.532414Z","caller":"traceutil/trace.go:172","msg":"trace[312429849] transaction","detail":"{read_only:false; response_revision:798; number_of_response:1; }","duration":"278.651598ms","start":"2026-03-17T20:06:52.253746Z","end":"2026-03-17T20:06:52.532397Z","steps":["trace[312429849] 'process raft request' (duration: 278.346424ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:52.532629Z","caller":"traceutil/trace.go:172","msg":"trace[74899417] transaction","detail":"{read_only:false; response_revision:800; number_of_response:1; }","duration":"274.430187ms","start":"2026-03-17T20:06:52.258190Z","end":"2026-03-17T20:06:52.532620Z","steps":["trace[74899417] 'process raft request' (duration: 274.173654ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:52.532626Z","caller":"traceutil/trace.go:172","msg":"trace[1013393057] transaction","detail":"{read_only:false; response_revision:799; number_of_response:1; }","duration":"274.494368ms","start":"2026-03-17T20:06:52.258083Z","end":"2026-03-17T20:06:52.532577Z","steps":["trace[1013393057] 'process raft request' (duration: 274.185723ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:52.532682Z","caller":"traceutil/trace.go:172","msg":"trace[789863945] transaction","detail":"{read_only:false; response_revision:802; number_of_response:1; }","duration":"269.806789ms","start":"2026-03-17T20:06:52.262844Z","end":"2026-03-17T20:06:52.532651Z","steps":["trace[789863945] 'process raft request' (duration: 269.733579ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:06:52.532763Z","caller":"traceutil/trace.go:172","msg":"trace[557096595] transaction","detail":"{read_only:false; response_revision:801; number_of_response:1; }","duration":"272.093412ms","start":"2026-03-17T20:06:52.260546Z","end":"2026-03-17T20:06:52.532639Z","steps":["trace[557096595] 'process raft request' (duration: 271.965921ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:53.011023Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"239.095223ms"} {"level":"warn","ts":"2026-03-17T20:06:53.011197Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"239.278576ms"} {"level":"warn","ts":"2026-03-17T20:06:54.129398Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980038,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:06:54.630345Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980038,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:06:54.777368Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"c1a97aa766b87c7e [logterm: 2, index: 880, vote: c1a97aa766b87c7e] ignored MsgPreVote from 81e3b46d999c74d3 [logterm: 2, index: 878] at term 2: lease is not expired (remaining ticks: 8)"} {"level":"warn","ts":"2026-03-17T20:06:55.131111Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980038,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:06:55.367465Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.849010746s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:55.367555Z","caller":"traceutil/trace.go:172","msg":"trace[2079525416] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:804; }","duration":"1.849115867s","start":"2026-03-17T20:06:53.518422Z","end":"2026-03-17T20:06:55.367538Z","steps":["trace[2079525416] 'range keys from in-memory index tree' (duration: 1.848944686s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:55.368092Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.356575606s","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-17T20:06:55.368704Z","caller":"traceutil/trace.go:172","msg":"trace[897423129] transaction","detail":"{read_only:false; response_revision:805; number_of_response:1; }","duration":"2.822042732s","start":"2026-03-17T20:06:52.546633Z","end":"2026-03-17T20:06:55.368675Z","steps":["trace[897423129] 'process raft request' (duration: 464.491402ms)","trace[897423129] 'compare' (duration: 2.355980938s)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:55.369067Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:52.546616Z","time spent":"2.822272436s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":423,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:55.369237Z","caller":"traceutil/trace.go:172","msg":"trace[426385887] transaction","detail":"{read_only:false; response_revision:806; number_of_response:1; }","duration":"2.821661016s","start":"2026-03-17T20:06:52.547530Z","end":"2026-03-17T20:06:55.369191Z","steps":["trace[426385887] 'process raft request' (duration: 2.821083337s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:55.369396Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:52.547510Z","time spent":"2.821786248s","remote":"127.0.0.1:47484","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4462,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:55.369833Z","caller":"traceutil/trace.go:172","msg":"trace[1358778407] transaction","detail":"{read_only:false; response_revision:807; number_of_response:1; }","duration":"2.822150054s","start":"2026-03-17T20:06:52.547663Z","end":"2026-03-17T20:06:55.369813Z","steps":["trace[1358778407] 'process raft request' (duration: 2.821310002s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:55.370228Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:52.547654Z","time spent":"2.822530529s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":427,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:55.370514Z","caller":"traceutil/trace.go:172","msg":"trace[13671116] transaction","detail":"{read_only:false; response_revision:808; number_of_response:1; }","duration":"2.819062569s","start":"2026-03-17T20:06:52.551433Z","end":"2026-03-17T20:06:55.370496Z","steps":["trace[13671116] 'process raft request' (duration: 2.818192346s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:55.371131Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:52.551412Z","time spent":"2.819683597s","remote":"127.0.0.1:47520","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4122,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:55.400096Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"2.125164017s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-17T20:06:55.400207Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.953606139s"} {"level":"warn","ts":"2026-03-17T20:06:55.400231Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.95363143s"} {"level":"info","ts":"2026-03-17T20:06:55.400605Z","caller":"traceutil/trace.go:172","msg":"trace[1484003796] transaction","detail":"{read_only:false; response_revision:809; number_of_response:1; }","duration":"2.125801407s","start":"2026-03-17T20:06:53.274768Z","end":"2026-03-17T20:06:55.400569Z","steps":["trace[1484003796] 'process raft request' (duration: 2.125552864s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:55.400862Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:53.274737Z","time spent":"2.12599631s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":493,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:55.628523Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000095626s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-17T20:06:55.628607Z","caller":"traceutil/trace.go:172","msg":"trace[1682443189] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.00032088s","start":"2026-03-17T20:06:53.628267Z","end":"2026-03-17T20:06:55.628588Z","steps":["trace[1682443189] 'agreement among raft nodes before linearized reading' (duration: 2.000091916s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:55.628666Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:53.628236Z","time spent":"2.000419851s","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/17 20:06:55 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-17T20:06:55.631352Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980038,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:06:56.132023Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980038,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:06:56.253994Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"653.735446ms"} {"level":"warn","ts":"2026-03-17T20:06:56.254045Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"653.795317ms"} {"level":"info","ts":"2026-03-17T20:06:56.254064Z","caller":"traceutil/trace.go:172","msg":"trace[2117101640] linearizableReadLoop","detail":"{readStateIndex:877; appliedIndex:878; }","duration":"2.625768189s","start":"2026-03-17T20:06:53.628279Z","end":"2026-03-17T20:06:56.254047Z","steps":["trace[2117101640] 'read index received' (duration: 2.625762669s)","trace[2117101640] 'applied index is now lower than readState.Index' (duration: 4.68µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:56.709861Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"455.725279ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-17T20:06:56.716924Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"262.867842ms"} {"level":"warn","ts":"2026-03-17T20:06:56.716959Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"262.906233ms"} {"level":"info","ts":"2026-03-17T20:06:56.717406Z","caller":"traceutil/trace.go:172","msg":"trace[1236492525] transaction","detail":"{read_only:false; response_revision:811; number_of_response:1; }","duration":"2.174913504s","start":"2026-03-17T20:06:54.542487Z","end":"2026-03-17T20:06:56.717400Z","steps":["trace[1236492525] 'process raft request' (duration: 2.174590878s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:56.717503Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:54.542460Z","time spent":"2.174987354s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":468,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:56.717368Z","caller":"traceutil/trace.go:172","msg":"trace[1269405795] transaction","detail":"{read_only:false; response_revision:813; number_of_response:1; }","duration":"1.338039528s","start":"2026-03-17T20:06:55.379316Z","end":"2026-03-17T20:06:56.717356Z","steps":["trace[1269405795] 'process raft request' (duration: 1.337872136s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:56.717698Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:55.379307Z","time spent":"1.338371942s","remote":"127.0.0.1:47484","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":4462,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:56.755097Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980047,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:06:56.865505Z","caller":"traceutil/trace.go:172","msg":"trace[1233614436] linearizableReadLoop","detail":"{readStateIndex:879; appliedIndex:882; }","duration":"611.377506ms","start":"2026-03-17T20:06:56.254110Z","end":"2026-03-17T20:06:56.865487Z","steps":["trace[1233614436] 'read index received' (duration: 611.368456ms)","trace[1233614436] 'applied index is now lower than readState.Index' (duration: 8.11µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.192987Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"127.39042ms"} {"level":"warn","ts":"2026-03-17T20:06:57.193055Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"127.465661ms"} {"level":"warn","ts":"2026-03-17T20:06:57.366781Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980048,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:06:57.516553Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"123.474489ms"} {"level":"warn","ts":"2026-03-17T20:06:57.516614Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"123.550021ms"} {"level":"warn","ts":"2026-03-17T20:06:57.662224Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.032993528s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:57.662328Z","caller":"traceutil/trace.go:172","msg":"trace[58456149] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:813; }","duration":"2.033112401s","start":"2026-03-17T20:06:55.629191Z","end":"2026-03-17T20:06:57.662304Z","steps":["trace[58456149] 'agreement among raft nodes before linearized reading' (duration: 1.236577832s)","trace[58456149] 'range keys from in-memory index tree' (duration: 796.379386ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.662383Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:55.629172Z","time spent":"2.033197813s","remote":"127.0.0.1:46490","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:06:57.662224Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.345871276s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/secrets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:06:57.662549Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.392929033s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/rolebindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:57.662555Z","caller":"traceutil/trace.go:172","msg":"trace[909362092] range","detail":"{range_begin:/registry/secrets; range_end:; response_count:0; response_revision:813; }","duration":"3.346226182s","start":"2026-03-17T20:06:54.316313Z","end":"2026-03-17T20:06:57.662539Z","steps":["trace[909362092] 'agreement among raft nodes before linearized reading' (duration: 2.549559081s)","trace[909362092] 'range keys from in-memory index tree' (duration: 796.247924ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.664459Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:54.316286Z","time spent":"3.348147879s","remote":"127.0.0.1:46668","response type":"/etcdserverpb.KV/Range","request count":0,"request size":21,"response count":0,"response size":28,"request content":"key:\"/registry/secrets\" limit:1 "} {"level":"info","ts":"2026-03-17T20:06:57.664836Z","caller":"traceutil/trace.go:172","msg":"trace[1005818989] range","detail":"{range_begin:/registry/rolebindings; range_end:; response_count:0; response_revision:813; }","duration":"2.395197285s","start":"2026-03-17T20:06:55.269613Z","end":"2026-03-17T20:06:57.664810Z","steps":["trace[1005818989] 'agreement among raft nodes before linearized reading' (duration: 1.596265952s)","trace[1005818989] 'range keys from in-memory index tree' (duration: 796.652201ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.662430Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.639594056s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:06:57.664900Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:55.269595Z","time spent":"2.395290606s","remote":"127.0.0.1:47224","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":28,"request content":"key:\"/registry/rolebindings\" limit:1 "} {"level":"info","ts":"2026-03-17T20:06:57.664952Z","caller":"traceutil/trace.go:172","msg":"trace[730746043] range","detail":"{range_begin:/registry/services/endpoints; range_end:; response_count:0; response_revision:813; }","duration":"2.642124332s","start":"2026-03-17T20:06:55.022801Z","end":"2026-03-17T20:06:57.664925Z","steps":["trace[730746043] 'agreement among raft nodes before linearized reading' (duration: 1.843038817s)","trace[730746043] 'range keys from in-memory index tree' (duration: 796.500529ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.665017Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:55.022786Z","time spent":"2.642213923s","remote":"127.0.0.1:46822","response type":"/etcdserverpb.KV/Range","request count":0,"request size":32,"response count":0,"response size":28,"request content":"key:\"/registry/services/endpoints\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:57.662791Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.285521136s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:57.665076Z","caller":"traceutil/trace.go:172","msg":"trace[1539812001] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:813; }","duration":"2.287802149s","start":"2026-03-17T20:06:55.377256Z","end":"2026-03-17T20:06:57.665058Z","steps":["trace[1539812001] 'agreement among raft nodes before linearized reading' (duration: 1.488299218s)","trace[1539812001] 'range keys from in-memory index tree' (duration: 797.198598ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.665139Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:55.377236Z","time spent":"2.28788198s","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:06:57.662836Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.97620117s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:57.665200Z","caller":"traceutil/trace.go:172","msg":"trace[477432621] range","detail":"{range_begin:/registry/horizontalpodautoscalers; range_end:; response_count:0; response_revision:813; }","duration":"1.978561255s","start":"2026-03-17T20:06:55.686626Z","end":"2026-03-17T20:06:57.665187Z","steps":["trace[477432621] 'agreement among raft nodes before linearized reading' (duration: 1.179029403s)","trace[477432621] 'range keys from in-memory index tree' (duration: 797.155036ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.665263Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:55.686611Z","time spent":"1.978638697s","remote":"127.0.0.1:46948","response type":"/etcdserverpb.KV/Range","request count":0,"request size":38,"response count":0,"response size":28,"request content":"key:\"/registry/horizontalpodautoscalers\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:57.662875Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.605385923s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/resourceslices\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:57.665487Z","caller":"traceutil/trace.go:172","msg":"trace[673993192] range","detail":"{range_begin:/registry/resourceslices; range_end:; response_count:0; response_revision:813; }","duration":"1.607991941s","start":"2026-03-17T20:06:56.057481Z","end":"2026-03-17T20:06:57.665473Z","steps":["trace[673993192] 'agreement among raft nodes before linearized reading' (duration: 808.273877ms)","trace[673993192] 'range keys from in-memory index tree' (duration: 797.103996ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.662312Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.294702899s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:06:57.663220Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.303415224s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:06:57.663610Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.233150149s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.19.213.226\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"warn","ts":"2026-03-17T20:06:57.663672Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"797.692145ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-17T20:06:57.665684Z","caller":"traceutil/trace.go:172","msg":"trace[1964102383] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:813; }","duration":"2.298059726s","start":"2026-03-17T20:06:55.367599Z","end":"2026-03-17T20:06:57.665659Z","steps":["trace[1964102383] 'agreement among raft nodes before linearized reading' (duration: 1.498241011s)","trace[1964102383] 'range keys from in-memory index tree' (duration: 796.453888ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.665675Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:56.057453Z","time spent":"1.608167583s","remote":"127.0.0.1:47692","response type":"/etcdserverpb.KV/Range","request count":0,"request size":28,"response count":0,"response size":28,"request content":"key:\"/registry/resourceslices\" limit:1 "} {"level":"info","ts":"2026-03-17T20:06:57.666035Z","caller":"traceutil/trace.go:172","msg":"trace[1466102103] transaction","detail":"{read_only:false; response_revision:814; number_of_response:1; }","duration":"2.205388722s","start":"2026-03-17T20:06:55.460631Z","end":"2026-03-17T20:06:57.666019Z","steps":["trace[1466102103] 'process raft request' (duration: 1.405236711s)","trace[1466102103] 'compare' (duration: 797.556033ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.666170Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:55.460595Z","time spent":"2.205484434s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:06:57.665735Z","caller":"traceutil/trace.go:172","msg":"trace[512879730] range","detail":"{range_begin:/registry/apiextensions.k8s.io/customresourcedefinitions; range_end:; response_count:0; response_revision:813; }","duration":"3.305948751s","start":"2026-03-17T20:06:54.359771Z","end":"2026-03-17T20:06:57.665720Z","steps":["trace[512879730] 'agreement among raft nodes before linearized reading' (duration: 2.506078064s)","trace[512879730] 'range keys from in-memory index tree' (duration: 797.31242ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.666401Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:54.359749Z","time spent":"3.306633569s","remote":"127.0.0.1:46590","response type":"/etcdserverpb.KV/Range","request count":0,"request size":60,"response count":0,"response size":28,"request content":"key:\"/registry/apiextensions.k8s.io/customresourcedefinitions\" limit:1 "} {"level":"info","ts":"2026-03-17T20:06:57.665773Z","caller":"traceutil/trace.go:172","msg":"trace[1944260149] range","detail":"{range_begin:/registry/masterleases/199.19.213.226; range_end:; response_count:1; response_revision:813; }","duration":"3.235302511s","start":"2026-03-17T20:06:54.430443Z","end":"2026-03-17T20:06:57.665746Z","steps":["trace[1944260149] 'agreement among raft nodes before linearized reading' (duration: 2.435416334s)","trace[1944260149] 'range keys from in-memory index tree' (duration: 796.952264ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.666523Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:54.430416Z","time spent":"3.236083852s","remote":"127.0.0.1:46550","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.19.213.226\" limit:1 "} {"level":"info","ts":"2026-03-17T20:06:57.778095Z","caller":"traceutil/trace.go:172","msg":"trace[859938704] linearizableReadLoop","detail":"{readStateIndex:883; appliedIndex:884; }","duration":"912.268699ms","start":"2026-03-17T20:06:56.865799Z","end":"2026-03-17T20:06:57.778068Z","steps":["trace[859938704] 'read index received' (duration: 912.260599ms)","trace[859938704] 'applied index is now lower than readState.Index' (duration: 6.55µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.891214Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.634376143s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/clusterroles\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:06:57.891269Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.160611804s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/limitranges\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:57.891315Z","caller":"traceutil/trace.go:172","msg":"trace[715551704] range","detail":"{range_begin:/registry/clusterroles; range_end:; response_count:0; response_revision:815; }","duration":"1.634482094s","start":"2026-03-17T20:06:56.256800Z","end":"2026-03-17T20:06:57.891282Z","steps":["trace[715551704] 'agreement among raft nodes before linearized reading' (duration: 1.521502774s)","trace[715551704] 'range keys from in-memory index tree' (duration: 112.849959ms)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:06:57.891346Z","caller":"traceutil/trace.go:172","msg":"trace[1201770377] range","detail":"{range_begin:/registry/limitranges; range_end:; response_count:0; response_revision:815; }","duration":"1.160703755s","start":"2026-03-17T20:06:56.730623Z","end":"2026-03-17T20:06:57.891327Z","steps":["trace[1201770377] 'agreement among raft nodes before linearized reading' (duration: 1.047669223s)","trace[1201770377] 'range keys from in-memory index tree' (duration: 112.91727ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.891382Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:56.256773Z","time spent":"1.634586615s","remote":"127.0.0.1:47256","response type":"/etcdserverpb.KV/Range","request count":0,"request size":26,"response count":0,"response size":28,"request content":"key:\"/registry/clusterroles\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:57.891410Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:56.730602Z","time spent":"1.160787947s","remote":"127.0.0.1:46758","response type":"/etcdserverpb.KV/Range","request count":0,"request size":25,"response count":0,"response size":28,"request content":"key:\"/registry/limitranges\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:57.891428Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.078362831s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true ","response":"range_response_count:68 size:5767"} {"level":"info","ts":"2026-03-17T20:06:57.891453Z","caller":"traceutil/trace.go:172","msg":"trace[946661638] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; response_count:68; response_revision:815; }","duration":"1.078390162s","start":"2026-03-17T20:06:56.813057Z","end":"2026-03-17T20:06:57.891447Z","steps":["trace[946661638] 'agreement among raft nodes before linearized reading' (duration: 965.218118ms)","trace[946661638] 'range keys from in-memory index tree' (duration: 112.86445ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:57.891473Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:56.813033Z","time spent":"1.078434793s","remote":"127.0.0.1:47634","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":68,"response size":5790,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" keys_only:true "} {"level":"warn","ts":"2026-03-17T20:06:57.972188Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"193.903751ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"warn","ts":"2026-03-17T20:06:57.972733Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:57.667853Z","time spent":"304.876952ms","remote":"127.0.0.1:46550","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"info","ts":"2026-03-17T20:06:57.973010Z","caller":"traceutil/trace.go:172","msg":"trace[1997522485] transaction","detail":"{read_only:false; response_revision:816; number_of_response:1; }","duration":"567.350807ms","start":"2026-03-17T20:06:57.405641Z","end":"2026-03-17T20:06:57.972992Z","steps":["trace[1997522485] 'process raft request' (duration: 566.922831ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:57.973223Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:57.405614Z","time spent":"567.429538ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":494,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:57.972713Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:57.387033Z","time spent":"585.66664ms","remote":"127.0.0.1:46640","response type":"/etcdserverpb.Lease/LeaseGrant","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""} {"level":"warn","ts":"2026-03-17T20:06:58.279311Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980055,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:06:58.290450Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"143.023852ms"} {"level":"info","ts":"2026-03-17T20:06:58.290545Z","caller":"traceutil/trace.go:172","msg":"trace[3713159] linearizableReadLoop","detail":"{readStateIndex:887; appliedIndex:888; }","duration":"512.27445ms","start":"2026-03-17T20:06:57.778200Z","end":"2026-03-17T20:06:58.290475Z","steps":["trace[3713159] 'read index received' (duration: 512.26396ms)","trace[3713159] 'applied index is now lower than readState.Index' (duration: 9.21µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:58.290575Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"143.152133ms"} {"level":"warn","ts":"2026-03-17T20:06:58.692120Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.316839158s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"warn","ts":"2026-03-17T20:06:58.692194Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.707713605s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:58.692225Z","caller":"traceutil/trace.go:172","msg":"trace[1839883995] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:816; }","duration":"1.316981939s","start":"2026-03-17T20:06:57.375222Z","end":"2026-03-17T20:06:58.692204Z","steps":["trace[1839883995] 'agreement among raft nodes before linearized reading' (duration: 915.378231ms)","trace[1839883995] 'range keys from in-memory index tree' (duration: 401.292534ms)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:06:58.692241Z","caller":"traceutil/trace.go:172","msg":"trace[302641292] range","detail":"{range_begin:/registry/leases; range_end:; response_count:0; response_revision:816; }","duration":"1.707747876s","start":"2026-03-17T20:06:56.984472Z","end":"2026-03-17T20:06:58.692220Z","steps":["trace[302641292] 'agreement among raft nodes before linearized reading' (duration: 1.306198378s)","trace[302641292] 'range keys from in-memory index tree' (duration: 401.502927ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:58.692274Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:57.375191Z","time spent":"1.31707385s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:58.692282Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:56.984370Z","time spent":"1.707903957s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":20,"response count":0,"response size":28,"request content":"key:\"/registry/leases\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:58.692563Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.664691756s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/networkpolicies\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:58.692586Z","caller":"traceutil/trace.go:172","msg":"trace[709682493] range","detail":"{range_begin:/registry/networkpolicies; range_end:; response_count:0; response_revision:816; }","duration":"1.664714746s","start":"2026-03-17T20:06:57.027864Z","end":"2026-03-17T20:06:58.692579Z","steps":["trace[709682493] 'agreement among raft nodes before linearized reading' (duration: 1.262782193s)","trace[709682493] 'range keys from in-memory index tree' (duration: 401.860792ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:58.692606Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:57.027831Z","time spent":"1.664770008s","remote":"127.0.0.1:47074","response type":"/etcdserverpb.KV/Range","request count":0,"request size":29,"response count":0,"response size":28,"request content":"key:\"/registry/networkpolicies\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:06:58.692112Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.056548115s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:58.693053Z","caller":"traceutil/trace.go:172","msg":"trace[389364717] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:816; }","duration":"1.057512358s","start":"2026-03-17T20:06:57.635528Z","end":"2026-03-17T20:06:58.693040Z","steps":["trace[389364717] 'agreement among raft nodes before linearized reading' (duration: 655.134469ms)","trace[389364717] 'range keys from in-memory index tree' (duration: 401.384646ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:58.693077Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:57.635508Z","time spent":"1.057561999s","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:06:58.693069Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.026153908s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:06:58.693016Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.317945384s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-17T20:06:58.693148Z","caller":"traceutil/trace.go:172","msg":"trace[1827327106] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:816; }","duration":"1.026231059s","start":"2026-03-17T20:06:57.666889Z","end":"2026-03-17T20:06:58.693120Z","steps":["trace[1827327106] 'agreement among raft nodes before linearized reading' (duration: 623.767198ms)","trace[1827327106] 'range keys from in-memory index tree' (duration: 402.36871ms)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:06:58.693180Z","caller":"traceutil/trace.go:172","msg":"trace[585051412] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:816; }","duration":"1.318107026s","start":"2026-03-17T20:06:57.375060Z","end":"2026-03-17T20:06:58.693167Z","steps":["trace[585051412] 'agreement among raft nodes before linearized reading' (duration: 915.573014ms)","trace[585051412] 'range keys from in-memory index tree' (duration: 402.311039ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:58.693286Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"402.593363ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure:<>>","response":"size:16"} {"level":"warn","ts":"2026-03-17T20:06:58.693291Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:57.375042Z","time spent":"1.318207847s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"info","ts":"2026-03-17T20:06:58.693594Z","caller":"traceutil/trace.go:172","msg":"trace[1437612500] transaction","detail":"{read_only:false; response_revision:818; number_of_response:1; }","duration":"718.139241ms","start":"2026-03-17T20:06:57.975441Z","end":"2026-03-17T20:06:58.693580Z","steps":["trace[1437612500] 'process raft request' (duration: 717.892557ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:58.693656Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:57.975422Z","time spent":"718.209072ms","remote":"127.0.0.1:46640","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":778,"response count":0,"response size":39,"request content":"compare: success:> failure:<>"} {"level":"warn","ts":"2026-03-17T20:06:58.695261Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"204.50536ms"} {"level":"warn","ts":"2026-03-17T20:06:58.695291Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"204.53761ms"} {"level":"info","ts":"2026-03-17T20:06:58.695319Z","caller":"traceutil/trace.go:172","msg":"trace[1233216691] linearizableReadLoop","detail":"{readStateIndex:890; appliedIndex:890; }","duration":"404.604211ms","start":"2026-03-17T20:06:58.290695Z","end":"2026-03-17T20:06:58.695299Z","steps":["trace[1233216691] 'read index received' (duration: 404.596061ms)","trace[1233216691] 'applied index is now lower than readState.Index' (duration: 6.9µs)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:06:58.695772Z","caller":"traceutil/trace.go:172","msg":"trace[131137673] transaction","detail":"{read_only:false; response_revision:819; number_of_response:1; }","duration":"720.130579ms","start":"2026-03-17T20:06:57.975544Z","end":"2026-03-17T20:06:58.695674Z","steps":["trace[131137673] 'process raft request' (duration: 719.904795ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:58.695923Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:57.975508Z","time spent":"720.319292ms","remote":"127.0.0.1:46550","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":120,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:06:58.697809Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"424.470346ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/certificatesigningrequests\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:58.697877Z","caller":"traceutil/trace.go:172","msg":"trace[1263509653] range","detail":"{range_begin:/registry/certificatesigningrequests; range_end:; response_count:0; response_revision:821; }","duration":"424.544977ms","start":"2026-03-17T20:06:58.273318Z","end":"2026-03-17T20:06:58.697863Z","steps":["trace[1263509653] 'agreement among raft nodes before linearized reading' (duration: 422.052571ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:58.697911Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:06:58.273297Z","time spent":"424.605249ms","remote":"127.0.0.1:47018","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":0,"response size":28,"request content":"key:\"/registry/certificatesigningrequests\" limit:1 "} {"level":"info","ts":"2026-03-17T20:06:58.950964Z","caller":"traceutil/trace.go:172","msg":"trace[1577615718] linearizableReadLoop","detail":"{readStateIndex:897; appliedIndex:897; }","duration":"242.106534ms","start":"2026-03-17T20:06:58.708841Z","end":"2026-03-17T20:06:58.950948Z","steps":["trace[1577615718] 'read index received' (duration: 242.101864ms)","trace[1577615718] 'applied index is now lower than readState.Index' (duration: 4.09µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:06:59.004784Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"295.927295ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:59.005138Z","caller":"traceutil/trace.go:172","msg":"trace[154685092] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:824; }","duration":"296.27559ms","start":"2026-03-17T20:06:58.708834Z","end":"2026-03-17T20:06:59.005110Z","steps":["trace[154685092] 'agreement among raft nodes before linearized reading' (duration: 242.349799ms)","trace[154685092] 'range keys from in-memory index tree' (duration: 53.1341ms)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:06:59.005243Z","caller":"traceutil/trace.go:172","msg":"trace[977190971] transaction","detail":"{read_only:false; response_revision:826; number_of_response:1; }","duration":"280.647147ms","start":"2026-03-17T20:06:58.724574Z","end":"2026-03-17T20:06:59.005222Z","steps":["trace[977190971] 'process raft request' (duration: 280.19254ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:59.009024Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"298.757985ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices/default/kubernetes\" limit:1 ","response":"range_response_count:1 size:519"} {"level":"info","ts":"2026-03-17T20:06:59.009065Z","caller":"traceutil/trace.go:172","msg":"trace[1358223066] range","detail":"{range_begin:/registry/endpointslices/default/kubernetes; range_end:; response_count:1; response_revision:827; }","duration":"298.812786ms","start":"2026-03-17T20:06:58.710244Z","end":"2026-03-17T20:06:59.009057Z","steps":["trace[1358223066] 'agreement among raft nodes before linearized reading' (duration: 298.669144ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:59.009047Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"216.281175ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingadmissionpolicies\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:59.009158Z","caller":"traceutil/trace.go:172","msg":"trace[221162976] range","detail":"{range_begin:/registry/validatingadmissionpolicies; range_end:; response_count:0; response_revision:827; }","duration":"216.400667ms","start":"2026-03-17T20:06:58.792729Z","end":"2026-03-17T20:06:59.009130Z","steps":["trace[221162976] 'agreement among raft nodes before linearized reading' (duration: 216.212604ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:06:59.649638Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"131.443907ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:06:59.649722Z","caller":"traceutil/trace.go:172","msg":"trace[577407650] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:835; }","duration":"131.540228ms","start":"2026-03-17T20:06:59.518167Z","end":"2026-03-17T20:06:59.649708Z","steps":["trace[577407650] 'agreement among raft nodes before linearized reading' (duration: 99.570662ms)","trace[577407650] 'range keys from in-memory index tree' (duration: 31.850715ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:07:06.695671Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"180.064069ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-17T20:07:06.695915Z","caller":"traceutil/trace.go:172","msg":"trace[768077652] linearizableReadLoop","detail":"{readStateIndex:978; appliedIndex:977; }","duration":"177.940709ms","start":"2026-03-17T20:07:06.517945Z","end":"2026-03-17T20:07:06.695886Z","steps":["trace[768077652] 'read index received' (duration: 545.758µs)","trace[768077652] 'applied index is now lower than readState.Index' (duration: 177.392671ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:07:06.696985Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"179.007694ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:06.697200Z","caller":"traceutil/trace.go:172","msg":"trace[354663732] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:898; }","duration":"179.201068ms","start":"2026-03-17T20:07:06.517903Z","end":"2026-03-17T20:07:06.697104Z","steps":["trace[354663732] 'agreement among raft nodes before linearized reading' (duration: 178.211774ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:08.134285Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980194,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:07:08.562311Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.349672924s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-17T20:07:08.562414Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.215413489s"} {"level":"warn","ts":"2026-03-17T20:07:08.562427Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"1.215429839s"} {"level":"warn","ts":"2026-03-17T20:07:08.635083Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980194,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:07:08.648096Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.128611451s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:08.648175Z","caller":"traceutil/trace.go:172","msg":"trace[807732135] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:901; }","duration":"1.128704462s","start":"2026-03-17T20:07:07.519456Z","end":"2026-03-17T20:07:08.648160Z","steps":["trace[807732135] 'range keys from in-memory index tree' (duration: 1.12856922s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:09.136225Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980194,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:07:09.361013Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"598.570433ms"} {"level":"warn","ts":"2026-03-17T20:07:09.361059Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"598.624374ms"} {"level":"info","ts":"2026-03-17T20:07:09.361265Z","caller":"traceutil/trace.go:172","msg":"trace[636449986] linearizableReadLoop","detail":"{readStateIndex:982; appliedIndex:982; }","duration":"1.727758828s","start":"2026-03-17T20:07:07.633470Z","end":"2026-03-17T20:07:09.361229Z","steps":["trace[636449986] 'read index received' (duration: 1.727752498s)","trace[636449986] 'applied index is now lower than readState.Index' (duration: 5.37µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:07:09.768183Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.134661017s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:09.768284Z","caller":"traceutil/trace.go:172","msg":"trace[2041817364] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:902; }","duration":"2.13479637s","start":"2026-03-17T20:07:07.633465Z","end":"2026-03-17T20:07:09.768262Z","steps":["trace[2041817364] 'agreement among raft nodes before linearized reading' (duration: 1.727958353s)","trace[2041817364] 'range keys from in-memory index tree' (duration: 406.664605ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:07:09.768408Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:07.633447Z","time spent":"2.134926901s","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:07:09.768944Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"407.643119ms","expected-duration":"100ms","prefix":"","request":"header: txn: success:> failure: >>","response":"size:16"} {"level":"info","ts":"2026-03-17T20:07:09.769211Z","caller":"traceutil/trace.go:172","msg":"trace[2071007052] transaction","detail":"{read_only:false; response_revision:904; number_of_response:1; }","duration":"1.673582888s","start":"2026-03-17T20:07:08.095610Z","end":"2026-03-17T20:07:09.769192Z","steps":["trace[2071007052] 'process raft request' (duration: 1.673468867s)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:07:09.769240Z","caller":"traceutil/trace.go:172","msg":"trace[815302596] transaction","detail":"{read_only:false; response_revision:903; number_of_response:1; }","duration":"1.952087907s","start":"2026-03-17T20:07:07.817139Z","end":"2026-03-17T20:07:09.769227Z","steps":["trace[815302596] 'process raft request' (duration: 1.544071883s)","trace[815302596] 'compare' (duration: 406.845747ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:07:09.769293Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:08.095576Z","time spent":"1.67367754s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":493,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:07:09.769328Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:07.817103Z","time spent":"1.952164557s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":677,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:07:09.773618Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"212.534807ms"} {"level":"warn","ts":"2026-03-17T20:07:09.773715Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"212.645629ms"} {"level":"info","ts":"2026-03-17T20:07:09.774500Z","caller":"traceutil/trace.go:172","msg":"trace[1114924256] transaction","detail":"{read_only:false; response_revision:906; number_of_response:1; }","duration":"641.996336ms","start":"2026-03-17T20:07:09.132135Z","end":"2026-03-17T20:07:09.774132Z","steps":["trace[1114924256] 'process raft request' (duration: 641.846615ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:09.776116Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:09.132117Z","time spent":"643.146483ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":467,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:07:09.779447Z","caller":"traceutil/trace.go:172","msg":"trace[34593150] linearizableReadLoop","detail":"{readStateIndex:984; appliedIndex:988; }","duration":"418.066834ms","start":"2026-03-17T20:07:09.361358Z","end":"2026-03-17T20:07:09.779425Z","steps":["trace[34593150] 'read index received' (duration: 418.057624ms)","trace[34593150] 'applied index is now lower than readState.Index' (duration: 7.89µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:07:09.779655Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"919.869528ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-17T20:07:09.779745Z","caller":"traceutil/trace.go:172","msg":"trace[1495112034] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:908; }","duration":"919.984269ms","start":"2026-03-17T20:07:08.859745Z","end":"2026-03-17T20:07:09.779729Z","steps":["trace[1495112034] 'agreement among raft nodes before linearized reading' (duration: 919.692335ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:09.779790Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:08.859732Z","time spent":"920.05006ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:07:09.779790Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"921.384118ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-17T20:07:09.779961Z","caller":"traceutil/trace.go:172","msg":"trace[184777628] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:908; }","duration":"921.582271ms","start":"2026-03-17T20:07:08.858363Z","end":"2026-03-17T20:07:09.779946Z","steps":["trace[184777628] 'agreement among raft nodes before linearized reading' (duration: 921.085325ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:09.780013Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:08.858330Z","time spent":"921.673974ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:07:09.780545Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"146.101219ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:09.780610Z","caller":"traceutil/trace.go:172","msg":"trace[133195962] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:908; }","duration":"146.16735ms","start":"2026-03-17T20:07:09.634427Z","end":"2026-03-17T20:07:09.780595Z","steps":["trace[133195962] 'agreement among raft nodes before linearized reading' (duration: 146.059859ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:09.779634Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.13137565s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:09.780697Z","caller":"traceutil/trace.go:172","msg":"trace[653126361] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:908; }","duration":"1.132456664s","start":"2026-03-17T20:07:08.648229Z","end":"2026-03-17T20:07:09.780686Z","steps":["trace[653126361] 'agreement among raft nodes before linearized reading' (duration: 1.131294309s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:16.874607Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"357.070993ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:16.874660Z","caller":"traceutil/trace.go:172","msg":"trace[1650486687] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:937; }","duration":"357.133054ms","start":"2026-03-17T20:07:16.517516Z","end":"2026-03-17T20:07:16.874649Z","steps":["trace[1650486687] 'range keys from in-memory index tree' (duration: 357.039322ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:07:16.958419Z","caller":"traceutil/trace.go:172","msg":"trace[1826508477] transaction","detail":"{read_only:false; response_revision:938; number_of_response:1; }","duration":"683.649592ms","start":"2026-03-17T20:07:16.274753Z","end":"2026-03-17T20:07:16.958402Z","steps":["trace[1826508477] 'process raft request' (duration: 683.50005ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:16.958537Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:16.274724Z","time spent":"683.757364ms","remote":"127.0.0.1:46858","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":18511,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:07:16.959488Z","caller":"traceutil/trace.go:172","msg":"trace[433154686] linearizableReadLoop","detail":"{readStateIndex:1023; appliedIndex:1024; }","duration":"332.739766ms","start":"2026-03-17T20:07:16.626736Z","end":"2026-03-17T20:07:16.959476Z","steps":["trace[433154686] 'read index received' (duration: 332.735836ms)","trace[433154686] 'applied index is now lower than readState.Index' (duration: 3.04µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:07:16.959858Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"333.121511ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:16.959905Z","caller":"traceutil/trace.go:172","msg":"trace[2137305109] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:938; }","duration":"333.171131ms","start":"2026-03-17T20:07:16.626724Z","end":"2026-03-17T20:07:16.959895Z","steps":["trace[2137305109] 'agreement among raft nodes before linearized reading' (duration: 333.09934ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:16.960558Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:16.626707Z","time spent":"333.843011ms","remote":"127.0.0.1:46490","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:07:17.898365Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"381.17492ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:17.898428Z","caller":"traceutil/trace.go:172","msg":"trace[659071674] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:938; }","duration":"381.24794ms","start":"2026-03-17T20:07:17.517168Z","end":"2026-03-17T20:07:17.898416Z","steps":["trace[659071674] 'range keys from in-memory index tree' (duration: 381.139589ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:18.126680Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980263,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:07:18.627857Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980263,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:07:19.128058Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980263,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:07:19.626647Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000493778s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context deadline exceeded"} {"level":"info","ts":"2026-03-17T20:07:19.626735Z","caller":"traceutil/trace.go:172","msg":"trace[1867985690] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.0005959s","start":"2026-03-17T20:07:17.626123Z","end":"2026-03-17T20:07:19.626719Z","steps":["trace[1867985690] 'agreement among raft nodes before linearized reading' (duration: 2.000490038s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:19.626795Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:17.626104Z","time spent":"2.000679631s","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:07:19.631721Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980263,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:07:20.132908Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980263,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:07:20.486593Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1101","msg":"c1a97aa766b87c7e [logterm: 2, index: 1035, vote: c1a97aa766b87c7e] ignored MsgPreVote from 81e3b46d999c74d3 [logterm: 2, index: 1025] at term 2: lease is not expired (remaining ticks: 4)"} {"level":"warn","ts":"2026-03-17T20:07:20.516576Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"3.187032964s","expected-duration":"1s"} {"level":"warn","ts":"2026-03-17T20:07:20.516754Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"3.169599372s"} {"level":"warn","ts":"2026-03-17T20:07:20.516779Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"3.169630532s"} {"level":"warn","ts":"2026-03-17T20:07:20.633392Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980263,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:07:20.664465Z","caller":"traceutil/trace.go:172","msg":"trace[334178207] linearizableReadLoop","detail":"{readStateIndex:1024; appliedIndex:1025; }","duration":"3.038313405s","start":"2026-03-17T20:07:17.626134Z","end":"2026-03-17T20:07:20.664447Z","steps":["trace[334178207] 'read index received' (duration: 3.038308025s)","trace[334178207] 'applied index is now lower than readState.Index' (duration: 4.33µs)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:07:20.664908Z","caller":"traceutil/trace.go:172","msg":"trace[63675622] transaction","detail":"{read_only:false; response_revision:940; number_of_response:1; }","duration":"2.842449232s","start":"2026-03-17T20:07:17.822450Z","end":"2026-03-17T20:07:20.664899Z","steps":["trace[63675622] 'process raft request' (duration: 2.842128658s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:20.665042Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:17.822384Z","time spent":"2.842552594s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":494,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:07:20.665066Z","caller":"traceutil/trace.go:172","msg":"trace[1316123623] transaction","detail":"{read_only:false; response_revision:941; number_of_response:1; }","duration":"2.833113153s","start":"2026-03-17T20:07:17.831943Z","end":"2026-03-17T20:07:20.665056Z","steps":["trace[1316123623] 'process raft request' (duration: 2.832807988s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:20.665118Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:17.831933Z","time spent":"2.833159153s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":468,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:07:20.665606Z","caller":"etcdserver/v3_server.go:888","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":8970780119931980268,"received-request-id":8970780119931980263} {"level":"info","ts":"2026-03-17T20:07:20.666214Z","caller":"traceutil/trace.go:172","msg":"trace[1653618592] transaction","detail":"{read_only:false; response_revision:949; number_of_response:1; }","duration":"481.516458ms","start":"2026-03-17T20:07:20.184679Z","end":"2026-03-17T20:07:20.666195Z","steps":["trace[1653618592] 'process raft request' (duration: 481.471567ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:07:20.667153Z","caller":"traceutil/trace.go:172","msg":"trace[1224722394] transaction","detail":"{read_only:false; response_revision:948; number_of_response:1; }","duration":"494.362396ms","start":"2026-03-17T20:07:20.172770Z","end":"2026-03-17T20:07:20.667132Z","steps":["trace[1224722394] 'process raft request' (duration: 493.351551ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:20.667251Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:20.172749Z","time spent":"494.457708ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":676,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:07:20.667342Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:20.184651Z","time spent":"481.63805ms","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":536,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:07:20.668766Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.77147749s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"warn","ts":"2026-03-17T20:07:20.668796Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.03588736s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:20.668813Z","caller":"traceutil/trace.go:172","msg":"trace[384164518] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:949; }","duration":"2.77152737s","start":"2026-03-17T20:07:17.897279Z","end":"2026-03-17T20:07:20.668806Z","steps":["trace[384164518] 'agreement among raft nodes before linearized reading' (duration: 2.771406339s)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:07:20.668821Z","caller":"traceutil/trace.go:172","msg":"trace[1839848995] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:949; }","duration":"1.03590583s","start":"2026-03-17T20:07:19.632906Z","end":"2026-03-17T20:07:20.668812Z","steps":["trace[1839848995] 'agreement among raft nodes before linearized reading' (duration: 1.03587965s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:20.668835Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:17.897260Z","time spent":"2.771569711s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":46,"response count":1,"response size":460,"request content":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:07:20.668843Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:19.632884Z","time spent":"1.035955462s","remote":"127.0.0.1:46490","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"warn","ts":"2026-03-17T20:07:20.668897Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.770439944s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:20.668922Z","caller":"traceutil/trace.go:172","msg":"trace[766201919] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:949; }","duration":"2.770464845s","start":"2026-03-17T20:07:17.898452Z","end":"2026-03-17T20:07:20.668917Z","steps":["trace[766201919] 'agreement among raft nodes before linearized reading' (duration: 2.770427514s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:20.668754Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"207.190487ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/volumeattachments\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:07:20.669179Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.765859431s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-17T20:07:20.669230Z","caller":"traceutil/trace.go:172","msg":"trace[554893050] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:949; }","duration":"2.765941992s","start":"2026-03-17T20:07:17.903276Z","end":"2026-03-17T20:07:20.669218Z","steps":["trace[554893050] 'agreement among raft nodes before linearized reading' (duration: 2.765448505s)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:07:20.668983Z","caller":"traceutil/trace.go:172","msg":"trace[660731797] range","detail":"{range_begin:/registry/volumeattachments; range_end:; response_count:0; response_revision:949; }","duration":"207.407969ms","start":"2026-03-17T20:07:20.461545Z","end":"2026-03-17T20:07:20.668953Z","steps":["trace[660731797] 'agreement among raft nodes before linearized reading' (duration: 207.168216ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:20.669310Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:17.903244Z","time spent":"2.766055773s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Range","request count":0,"request size":48,"response count":1,"response size":464,"request content":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:07:21.077266Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"150.876416ms"} {"level":"warn","ts":"2026-03-17T20:07:21.077348Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"150.972208ms"} {"level":"warn","ts":"2026-03-17T20:07:21.170855Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980269,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:07:21.671854Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980269,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:07:22.147380Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1275","msg":"c1a97aa766b87c7e stepped down to follower since quorum is not active"} {"level":"info","ts":"2026-03-17T20:07:22.147455Z","logger":"raft","caller":"v3@v3.6.0/raft.go:897","msg":"c1a97aa766b87c7e became follower at term 2"} {"level":"info","ts":"2026-03-17T20:07:22.147466Z","logger":"raft","caller":"v3@v3.6.0/node.go:376","msg":"raft.node: c1a97aa766b87c7e lost leader c1a97aa766b87c7e at term 2"} {"level":"warn","ts":"2026-03-17T20:07:22.172881Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980269,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:07:22.172992Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"warn","ts":"2026-03-17T20:07:22.673567Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980269,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:07:22.673636Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"warn","ts":"2026-03-17T20:07:22.680509Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.000011848s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-17T20:07:22.680578Z","caller":"traceutil/trace.go:172","msg":"trace[1909642592] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"2.000087589s","start":"2026-03-17T20:07:20.680476Z","end":"2026-03-17T20:07:22.680564Z","steps":["trace[1909642592] 'agreement among raft nodes before linearized reading' (duration: 2.000009888s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:22.680655Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:20.680470Z","time spent":"2.00017687s","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/17 20:07:22 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"warn","ts":"2026-03-17T20:07:23.174584Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980269,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:07:23.174663Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"warn","ts":"2026-03-17T20:07:23.675081Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980269,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:07:23.675221Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"info","ts":"2026-03-17T20:07:24.016717Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"info","ts":"2026-03-17T20:07:24.016817Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"info","ts":"2026-03-17T20:07:24.016836Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"info","ts":"2026-03-17T20:07:24.016855Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"info","ts":"2026-03-17T20:07:24.016863Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"info","ts":"2026-03-17T20:07:24.016877Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"info","ts":"2026-03-17T20:07:24.016887Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"warn","ts":"2026-03-17T20:07:24.068252Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.34162737s","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"warn","ts":"2026-03-17T20:07:24.068804Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"3.140439166s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 serializable:true keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:24.068841Z","caller":"traceutil/trace.go:172","msg":"trace[1532027152] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:949; }","duration":"3.140482367s","start":"2026-03-17T20:07:20.928343Z","end":"2026-03-17T20:07:24.068826Z","steps":["trace[1532027152] 'range keys from in-memory index tree' (duration: 3.140425776s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:24.175635Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980269,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:07:24.175703Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"info","ts":"2026-03-17T20:07:24.176518Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"warn","ts":"2026-03-17T20:07:24.676080Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980269,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:07:24.676166Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"info","ts":"2026-03-17T20:07:24.676892Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1761","msg":"c1a97aa766b87c7e no leader at term 2; dropping index reading msg"} {"level":"warn","ts":"2026-03-17T20:07:24.680773Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.999789659s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"","error":"context canceled"} {"level":"info","ts":"2026-03-17T20:07:24.680815Z","caller":"traceutil/trace.go:172","msg":"trace[1650062465] range","detail":"{range_begin:/registry/health; range_end:; }","duration":"1.9998438s","start":"2026-03-17T20:07:22.680961Z","end":"2026-03-17T20:07:24.680805Z","steps":["trace[1650062465] 'agreement among raft nodes before linearized reading' (duration: 1.999788149s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:24.680861Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:22.680950Z","time spent":"1.99989111s","remote":"127.0.0.1:46490","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":0,"request content":"key:\"/registry/health\" "} 2026/03/17 20:07:24 WARNING: [core] [Server #6]grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing" {"level":"info","ts":"2026-03-17T20:07:25.147090Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"c1a97aa766b87c7e is starting a new election at term 2"} {"level":"info","ts":"2026-03-17T20:07:25.148590Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"c1a97aa766b87c7e became pre-candidate at term 2"} {"level":"info","ts":"2026-03-17T20:07:25.149963Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"c1a97aa766b87c7e [logterm: 2, index: 1040] sent MsgPreVote request to 81e3b46d999c74d3 at term 2"} {"level":"info","ts":"2026-03-17T20:07:25.150099Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"c1a97aa766b87c7e [logterm: 2, index: 1040] sent MsgPreVote request to c6d6ded81dced554 at term 2"} {"level":"info","ts":"2026-03-17T20:07:25.150182Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"c1a97aa766b87c7e received MsgPreVoteResp from c1a97aa766b87c7e at term 2"} {"level":"info","ts":"2026-03-17T20:07:25.150298Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"c1a97aa766b87c7e has received 1 MsgPreVoteResp votes and 0 vote rejections"} {"level":"warn","ts":"2026-03-17T20:07:25.176604Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980269,"retry-timeout":"500ms"} {"level":"info","ts":"2026-03-17T20:07:25.220041Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"c1a97aa766b87c7e received MsgPreVoteResp from c6d6ded81dced554 at term 2"} {"level":"info","ts":"2026-03-17T20:07:25.220121Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"c1a97aa766b87c7e has received 2 MsgPreVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-17T20:07:25.220152Z","logger":"raft","caller":"v3@v3.6.0/raft.go:912","msg":"c1a97aa766b87c7e became candidate at term 3"} {"level":"info","ts":"2026-03-17T20:07:25.220171Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"c1a97aa766b87c7e [logterm: 2, index: 1040] sent MsgVote request to 81e3b46d999c74d3 at term 3"} {"level":"info","ts":"2026-03-17T20:07:25.220189Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"c1a97aa766b87c7e [logterm: 2, index: 1040] sent MsgVote request to c6d6ded81dced554 at term 3"} {"level":"info","ts":"2026-03-17T20:07:25.569607Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgAppResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"warn","ts":"2026-03-17T20:07:25.676853Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980269,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:07:26.178525Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8970780119931980269,"retry-timeout":"500ms"} {"level":"warn","ts":"2026-03-17T20:07:26.326514Z","caller":"wal/wal.go:845","msg":"slow fdatasync","took":"1.10623032s","expected-duration":"1s"} {"level":"info","ts":"2026-03-17T20:07:26.326596Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"c1a97aa766b87c7e received MsgVoteResp from c1a97aa766b87c7e at term 3"} {"level":"info","ts":"2026-03-17T20:07:26.326613Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"c1a97aa766b87c7e has received 1 MsgVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-17T20:07:26.328354Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328401Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328412Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328424Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328438Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328450Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328460Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328471Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328483Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328497Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328506Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328522Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328530Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328540Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328556Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgHeartbeatResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328579Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1175","msg":"c1a97aa766b87c7e [term: 3] ignored a MsgAppResp message with lower term from 81e3b46d999c74d3 [term: 2]"} {"level":"info","ts":"2026-03-17T20:07:26.328619Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1251","msg":"c1a97aa766b87c7e [logterm: 2, index: 1040, vote: c1a97aa766b87c7e] rejected MsgPreVote from c6d6ded81dced554 [logterm: 2, index: 1040] at term 3"} {"level":"info","ts":"2026-03-17T20:07:26.544791Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"c1a97aa766b87c7e received MsgVoteResp from c6d6ded81dced554 at term 3"} {"level":"info","ts":"2026-03-17T20:07:26.544857Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"c1a97aa766b87c7e has received 2 MsgVoteResp votes and 0 vote rejections"} {"level":"info","ts":"2026-03-17T20:07:26.544886Z","logger":"raft","caller":"v3@v3.6.0/raft.go:970","msg":"c1a97aa766b87c7e became leader at term 3"} {"level":"info","ts":"2026-03-17T20:07:26.544901Z","logger":"raft","caller":"v3@v3.6.0/node.go:370","msg":"raft.node: c1a97aa766b87c7e elected leader c1a97aa766b87c7e at term 3"} {"level":"warn","ts":"2026-03-17T20:07:26.545148Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"5.874821715s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: leader changed"} {"level":"info","ts":"2026-03-17T20:07:26.545240Z","caller":"traceutil/trace.go:172","msg":"trace[1580944199] range","detail":"{range_begin:; range_end:; }","duration":"5.874936526s","start":"2026-03-17T20:07:20.670287Z","end":"2026-03-17T20:07:26.545223Z","steps":["trace[1580944199] 'agreement among raft nodes before linearized reading' (duration: 5.874816705s)"],"step_count":1} {"level":"error","ts":"2026-03-17T20:07:26.545337Z","caller":"etcdhttp/health.go:345","msg":"Health check error","path":"/readyz","reason":"[-]linearizable_read failed: etcdserver: leader changed\n[+]non_learner ok\n[+]data_corruption ok\n[+]serializable_read ok\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHTTPEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:345\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2220\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2747\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3210\nnet/http.(*conn).serve\n\tnet/http/server.go:2092"} {"level":"info","ts":"2026-03-17T20:07:26.617264Z","caller":"traceutil/trace.go:172","msg":"trace[83294044] transaction","detail":"{read_only:false; response_revision:953; number_of_response:1; }","duration":"3.945390491s","start":"2026-03-17T20:07:22.671850Z","end":"2026-03-17T20:07:26.617241Z","steps":["trace[83294044] 'process raft request' (duration: 3.94534526s)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:07:26.617276Z","caller":"traceutil/trace.go:172","msg":"trace[1043078523] transaction","detail":"{read_only:false; response_revision:951; number_of_response:1; }","duration":"5.934732183s","start":"2026-03-17T20:07:20.682495Z","end":"2026-03-17T20:07:26.617227Z","steps":["trace[1043078523] 'process raft request' (duration: 5.934585291s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:26.617428Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:22.671640Z","time spent":"3.945706865s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":468,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:07:26.617404Z","caller":"traceutil/trace.go:172","msg":"trace[1124733304] transaction","detail":"{read_only:false; response_revision:950; number_of_response:1; }","duration":"5.938039209s","start":"2026-03-17T20:07:20.679335Z","end":"2026-03-17T20:07:26.617374Z","steps":["trace[1124733304] 'process raft request' (duration: 5.93738846s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:26.617592Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:20.682487Z","time spent":"5.934946635s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":427,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"info","ts":"2026-03-17T20:07:26.617751Z","caller":"traceutil/trace.go:172","msg":"trace[64441712] transaction","detail":"{read_only:false; response_revision:952; number_of_response:1; }","duration":"3.94757345s","start":"2026-03-17T20:07:22.670162Z","end":"2026-03-17T20:07:26.617735Z","steps":["trace[64441712] 'process raft request' (duration: 3.946995413s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:26.617809Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:20.679323Z","time spent":"5.938269041s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":423,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:07:26.617858Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:22.670129Z","time spent":"3.947693182s","remote":"127.0.0.1:47028","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":494,"response count":0,"response size":39,"request content":"compare: success:> failure: >"} {"level":"warn","ts":"2026-03-17T20:07:26.618224Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"1.928052612s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:26.618291Z","caller":"traceutil/trace.go:172","msg":"trace[1827562119] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:953; }","duration":"1.928136583s","start":"2026-03-17T20:07:24.690138Z","end":"2026-03-17T20:07:26.618275Z","steps":["trace[1827562119] 'agreement among raft nodes before linearized reading' (duration: 1.928010131s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:26.618286Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"5.180603944s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/poddisruptionbudgets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:26.618335Z","caller":"traceutil/trace.go:172","msg":"trace[1972676265] range","detail":"{range_begin:/registry/poddisruptionbudgets; range_end:; response_count:0; response_revision:953; }","duration":"5.180653774s","start":"2026-03-17T20:07:21.437669Z","end":"2026-03-17T20:07:26.618322Z","steps":["trace[1972676265] 'agreement among raft nodes before linearized reading' (duration: 5.180552113s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:26.618392Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:21.437646Z","time spent":"5.180738605s","remote":"127.0.0.1:47188","response type":"/etcdserverpb.KV/Range","request count":0,"request size":34,"response count":0,"response size":28,"request content":"key:\"/registry/poddisruptionbudgets\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:07:26.618394Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"5.330631399s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/poddisruptionbudgets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:07:26.618542Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"5.943621435s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/kube-system/kube-apiserver-controller-1.189db9b58b3cea6e\" limit:1 ","response":"range_response_count:1 size:794"} {"level":"info","ts":"2026-03-17T20:07:26.618556Z","caller":"traceutil/trace.go:172","msg":"trace[220530690] range","detail":"{range_begin:/registry/poddisruptionbudgets; range_end:; response_count:0; response_revision:953; }","duration":"5.330786902s","start":"2026-03-17T20:07:21.287752Z","end":"2026-03-17T20:07:26.618539Z","steps":["trace[220530690] 'agreement among raft nodes before linearized reading' (duration: 5.330602849s)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:07:26.618587Z","caller":"traceutil/trace.go:172","msg":"trace[791945440] range","detail":"{range_begin:/registry/events/kube-system/kube-apiserver-controller-1.189db9b58b3cea6e; range_end:; response_count:1; response_revision:953; }","duration":"5.943668345s","start":"2026-03-17T20:07:20.674906Z","end":"2026-03-17T20:07:26.618574Z","steps":["trace[791945440] 'agreement among raft nodes before linearized reading' (duration: 5.943514032s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:26.618619Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:20.674892Z","time spent":"5.943719567s","remote":"127.0.0.1:46640","response type":"/etcdserverpb.KV/Range","request count":0,"request size":77,"response count":1,"response size":817,"request content":"key:\"/registry/events/kube-system/kube-apiserver-controller-1.189db9b58b3cea6e\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:07:26.618601Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:21.287731Z","time spent":"5.330856523s","remote":"127.0.0.1:47188","response type":"/etcdserverpb.KV/Range","request count":0,"request size":34,"response count":0,"response size":28,"request content":"key:\"/registry/poddisruptionbudgets\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:07:26.618687Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.136636181s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/199.19.213.226\" limit:1 ","response":"range_response_count:1 size:135"} {"level":"info","ts":"2026-03-17T20:07:26.618737Z","caller":"traceutil/trace.go:172","msg":"trace[1241634371] range","detail":"{range_begin:/registry/masterleases/199.19.213.226; range_end:; response_count:1; response_revision:953; }","duration":"2.136684981s","start":"2026-03-17T20:07:24.482042Z","end":"2026-03-17T20:07:26.618727Z","steps":["trace[1241634371] 'agreement among raft nodes before linearized reading' (duration: 2.136500509s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:26.618765Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:24.482024Z","time spent":"2.136733732s","remote":"127.0.0.1:46550","response type":"/etcdserverpb.KV/Range","request count":0,"request size":41,"response count":1,"response size":158,"request content":"key:\"/registry/masterleases/199.19.213.226\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:07:26.618965Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.573331768s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:26.619005Z","caller":"traceutil/trace.go:172","msg":"trace[2077177224] range","detail":"{range_begin:/registry/validatingadmissionpolicybindings; range_end:; response_count:0; response_revision:953; }","duration":"2.573372339s","start":"2026-03-17T20:07:24.045623Z","end":"2026-03-17T20:07:26.618995Z","steps":["trace[2077177224] 'agreement among raft nodes before linearized reading' (duration: 2.573298658s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:26.619032Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:24.045592Z","time spent":"2.5734336s","remote":"127.0.0.1:47616","response type":"/etcdserverpb.KV/Range","request count":0,"request size":47,"response count":0,"response size":28,"request content":"key:\"/registry/validatingadmissionpolicybindings\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:07:26.619202Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"2.821753158s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/daemonsets\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:26.619241Z","caller":"traceutil/trace.go:172","msg":"trace[775693685] range","detail":"{range_begin:/registry/daemonsets; range_end:; response_count:0; response_revision:953; }","duration":"2.821791528s","start":"2026-03-17T20:07:23.797440Z","end":"2026-03-17T20:07:26.619231Z","steps":["trace[775693685] 'agreement among raft nodes before linearized reading' (duration: 2.821730818s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:26.619268Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:23.797405Z","time spent":"2.82185619s","remote":"127.0.0.1:47508","response type":"/etcdserverpb.KV/Range","request count":0,"request size":24,"response count":0,"response size":28,"request content":"key:\"/registry/daemonsets\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:07:26.619429Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"4.343739558s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/cronjobs\" limit:1 ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:26.619493Z","caller":"traceutil/trace.go:172","msg":"trace[1087827906] range","detail":"{range_begin:/registry/cronjobs; range_end:; response_count:0; response_revision:953; }","duration":"4.343804428s","start":"2026-03-17T20:07:22.275674Z","end":"2026-03-17T20:07:26.619479Z","steps":["trace[1087827906] 'agreement among raft nodes before linearized reading' (duration: 4.343705297s)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:26.619535Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:22.275653Z","time spent":"4.343870109s","remote":"127.0.0.1:46996","response type":"/etcdserverpb.KV/Range","request count":0,"request size":22,"response count":0,"response size":28,"request content":"key:\"/registry/cronjobs\" limit:1 "} {"level":"warn","ts":"2026-03-17T20:07:26.618333Z","caller":"v3rpc/interceptor.go:202","msg":"request stats","start time":"2026-03-17T20:07:24.690119Z","time spent":"1.928203623s","remote":"127.0.0.1:46516","response type":"/etcdserverpb.KV/Range","request count":0,"request size":18,"response count":0,"response size":28,"request content":"key:\"/registry/health\" "} {"level":"info","ts":"2026-03-17T20:07:28.732246Z","caller":"traceutil/trace.go:172","msg":"trace[880196097] linearizableReadLoop","detail":"{readStateIndex:1081; appliedIndex:1082; }","duration":"106.688485ms","start":"2026-03-17T20:07:28.625497Z","end":"2026-03-17T20:07:28.732186Z","steps":["trace[880196097] 'read index received' (duration: 106.680135ms)","trace[880196097] 'applied index is now lower than readState.Index' (duration: 6.96µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:07:28.821238Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"195.434655ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"warn","ts":"2026-03-17T20:07:28.821305Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"195.80627ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-cp-lock\" limit:1 ","response":"range_response_count:1 size:437"} {"level":"info","ts":"2026-03-17T20:07:28.821329Z","caller":"traceutil/trace.go:172","msg":"trace[168122712] range","detail":"{range_begin:/registry/leases/kube-system/plndr-cp-lock; range_end:; response_count:1; response_revision:988; }","duration":"195.827431ms","start":"2026-03-17T20:07:28.625494Z","end":"2026-03-17T20:07:28.821322Z","steps":["trace[168122712] 'agreement among raft nodes before linearized reading' (duration: 106.857997ms)","trace[168122712] 'range keys from in-memory index tree' (duration: 88.914793ms)"],"step_count":2} {"level":"info","ts":"2026-03-17T20:07:28.821303Z","caller":"traceutil/trace.go:172","msg":"trace[1494860643] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:988; }","duration":"195.507726ms","start":"2026-03-17T20:07:28.625780Z","end":"2026-03-17T20:07:28.821288Z","steps":["trace[1494860643] 'agreement among raft nodes before linearized reading' (duration: 193.966775ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:07:28.821343Z","caller":"traceutil/trace.go:172","msg":"trace[749040724] transaction","detail":"{read_only:false; response_revision:989; number_of_response:1; }","duration":"195.729259ms","start":"2026-03-17T20:07:28.625587Z","end":"2026-03-17T20:07:28.821317Z","steps":["trace[749040724] 'process raft request' (duration: 194.139648ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:28.821237Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"190.914843ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/leases/kube-system/plndr-svcs-lock\" limit:1 ","response":"range_response_count:1 size:441"} {"level":"info","ts":"2026-03-17T20:07:28.821513Z","caller":"traceutil/trace.go:172","msg":"trace[1528258316] range","detail":"{range_begin:/registry/leases/kube-system/plndr-svcs-lock; range_end:; response_count:1; response_revision:988; }","duration":"191.197887ms","start":"2026-03-17T20:07:28.630309Z","end":"2026-03-17T20:07:28.821506Z","steps":["trace[1528258316] 'agreement among raft nodes before linearized reading' (duration: 189.456443ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:32.786713Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"c6d6ded81dced554","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"40.021347ms"} {"level":"warn","ts":"2026-03-17T20:07:32.787418Z","caller":"etcdserver/raft.go:387","msg":"leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk","to":"81e3b46d999c74d3","heartbeat-interval":"100ms","expected-duration":"200ms","exceeded-duration":"40.767437ms"} {"level":"info","ts":"2026-03-17T20:07:32.796100Z","caller":"traceutil/trace.go:172","msg":"trace[463721414] linearizableReadLoop","detail":"{readStateIndex:1180; appliedIndex:1181; }","duration":"168.121132ms","start":"2026-03-17T20:07:32.627950Z","end":"2026-03-17T20:07:32.796071Z","steps":["trace[463721414] 'read index received' (duration: 168.110462ms)","trace[463721414] 'applied index is now lower than readState.Index' (duration: 9.28µs)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:07:32.796447Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"168.384705ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/health\" ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:32.796496Z","caller":"traceutil/trace.go:172","msg":"trace[862359386] range","detail":"{range_begin:/registry/health; range_end:; response_count:0; response_revision:1068; }","duration":"168.548607ms","start":"2026-03-17T20:07:32.627934Z","end":"2026-03-17T20:07:32.796483Z","steps":["trace[862359386] 'agreement among raft nodes before linearized reading' (duration: 168.242553ms)"],"step_count":1} {"level":"info","ts":"2026-03-17T20:07:32.797223Z","caller":"traceutil/trace.go:172","msg":"trace[1064626136] transaction","detail":"{read_only:false; response_revision:1069; number_of_response:1; }","duration":"155.581801ms","start":"2026-03-17T20:07:32.641367Z","end":"2026-03-17T20:07:32.796949Z","steps":["trace[1064626136] 'process raft request' (duration: 154.920842ms)"],"step_count":1} {"level":"warn","ts":"2026-03-17T20:07:34.389173Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:59068","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:07:37.118244Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:59090","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:07:37.129190Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:59102","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:07:37.159197Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:59126","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:07:37.184853Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:59136","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:07:37.277720Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:59146","server-name":"","error":"EOF"} {"level":"info","ts":"2026-03-17T20:07:49.020629Z","caller":"traceutil/trace.go:172","msg":"trace[349023074] transaction","detail":"{read_only:false; response_revision:1248; number_of_response:1; }","duration":"107.269391ms","start":"2026-03-17T20:07:48.913333Z","end":"2026-03-17T20:07:49.020602Z","steps":["trace[349023074] 'process raft request' (duration: 92.20233ms)","trace[349023074] 'compare' (duration: 14.793867ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:07:49.695257Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"115.424107ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"info","ts":"2026-03-17T20:07:49.695777Z","caller":"traceutil/trace.go:172","msg":"trace[1251822467] linearizableReadLoop","detail":"{readStateIndex:1407; appliedIndex:1406; }","duration":"178.00518ms","start":"2026-03-17T20:07:49.517727Z","end":"2026-03-17T20:07:49.695732Z","steps":["trace[1251822467] 'read index received' (duration: 62.997619ms)","trace[1251822467] 'applied index is now lower than readState.Index' (duration: 115.003511ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:07:49.865227Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"169.832133ms","expected-duration":"100ms","prefix":"","request":"header: lease_revoke:","response":"size:28"} {"level":"warn","ts":"2026-03-17T20:07:49.865252Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"347.529029ms","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"range_response_count:0 size:5"} {"level":"info","ts":"2026-03-17T20:07:49.865468Z","caller":"traceutil/trace.go:172","msg":"trace[1906670568] range","detail":"{range_begin:; range_end:; response_count:0; response_revision:1250; }","duration":"347.763442ms","start":"2026-03-17T20:07:49.517675Z","end":"2026-03-17T20:07:49.865438Z","steps":["trace[1906670568] 'agreement among raft nodes before linearized reading' (duration: 178.157502ms)","trace[1906670568] 'range keys from in-memory index tree' (duration: 169.347466ms)"],"step_count":2} {"level":"warn","ts":"2026-03-17T20:08:00.123087Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42680","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:08:00.140645Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42690","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:08:00.153197Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42706","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:08:00.166807Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42724","server-name":"","error":"EOF"} {"level":"warn","ts":"2026-03-17T20:08:00.183291Z","caller":"embed/config_logging.go:188","msg":"rejected connection on client endpoint","remote-addr":"127.0.0.1:42740","server-name":"","error":"EOF"}