------------------------------------------------------------------------------- NGINX Ingress controller Release: v1.12.1 Build: 51c2b819690bbf1709b844dbf321a9acf6eda5a7 Repository: https://github.com/kubernetes/ingress-nginx nginx version: nginx/1.25.5 ------------------------------------------------------------------------------- W0721 05:26:18.069926 7 client_config.go:667] Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work. I0721 05:26:18.070570 7 main.go:205] "Creating API client" host="https://10.96.0.1:443" I0721 05:26:18.080139 7 main.go:248] "Running in Kubernetes cluster" major="1" minor="28" git="v1.28.13" state="clean" commit="024ab2a13ff700705efd3ab63a0e27c38c462070" platform="linux/amd64" I0721 05:26:18.083127 7 main.go:83] "Valid default backend" service="ingress-nginx/ingress-nginx-defaultbackend" I0721 05:26:18.311291 7 main.go:101] "SSL fake certificate created" file="/etc/ingress-controller/ssl/default-fake-certificate.pem" I0721 05:26:18.326970 7 ssl.go:535] "loading tls certificate" path="/usr/local/certificates/cert" key="/usr/local/certificates/key" I0721 05:26:18.339415 7 nginx.go:271] "Starting NGINX Ingress controller" I0721 05:26:18.344643 7 event.go:377] Event(v1.ObjectReference{Kind:"ConfigMap", Namespace:"ingress-nginx", Name:"ingress-nginx-controller", UID:"f572ba9b-9366-4a7e-aa62-bc469394e32a", APIVersion:"v1", ResourceVersion:"1308", FieldPath:""}): type: 'Normal' reason: 'CREATE' ConfigMap ingress-nginx/ingress-nginx-controller I0721 05:26:18.350276 7 event.go:377] Event(v1.ObjectReference{Kind:"ConfigMap", Namespace:"ingress-nginx", Name:"ingress-nginx-tcp", UID:"fc60e245-ec92-45b4-9b10-d3621576e673", APIVersion:"v1", ResourceVersion:"1310", FieldPath:""}): type: 'Normal' reason: 'CREATE' ConfigMap ingress-nginx/ingress-nginx-tcp I0721 05:26:18.350329 7 event.go:377] Event(v1.ObjectReference{Kind:"ConfigMap", Namespace:"ingress-nginx", Name:"ingress-nginx-udp", UID:"43d9b52b-8625-490a-a5db-92619f282039", APIVersion:"v1", ResourceVersion:"1307", FieldPath:""}): type: 'Normal' reason: 'CREATE' ConfigMap ingress-nginx/ingress-nginx-udp I0721 05:26:19.542322 7 leaderelection.go:257] attempting to acquire leader lease ingress-nginx/ingress-nginx-leader... I0721 05:26:19.542726 7 nginx.go:317] "Starting NGINX process" I0721 05:26:19.543383 7 nginx.go:337] "Starting validation webhook" address=":7443" certPath="/usr/local/certificates/cert" keyPath="/usr/local/certificates/key" W0721 05:26:19.543586 7 controller.go:603] Service "ingress-nginx/ingress-nginx-defaultbackend" does not have any active Endpoint W0721 05:26:19.543632 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:19.543651 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0721 05:26:19.543727 7 controller.go:196] "Configuration changes detected, backend reload required" I0721 05:26:19.552594 7 leaderelection.go:271] successfully acquired lease ingress-nginx/ingress-nginx-leader I0721 05:26:19.552676 7 status.go:85] "New leader elected" identity="ingress-nginx-controller-vw7wm" I0721 05:26:19.577546 7 controller.go:216] "Backend successfully reloaded" I0721 05:26:19.577701 7 controller.go:227] "Initial sync, sleeping for 1 second" I0721 05:26:19.577823 7 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-vw7wm", UID:"a91d7e05-6640-418e-825c-ea1b85e5c9c4", APIVersion:"v1", ResourceVersion:"1410", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration W0721 05:26:23.357983 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:23.358011 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:26.691818 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:26.691866 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:30.026136 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:30.026186 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:33.358110 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:33.358187 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:36.692203 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:36.692289 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:40.025457 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:40.025498 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:43.359197 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:43.359227 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:50.505827 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:50.505867 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:56.170544 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:26:56.170590 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:27:45.430790 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:27:45.430828 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:27:48.765196 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:27:48.765255 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:27:53.396268 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:27:53.396294 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:27:56.731292 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:27:56.731326 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:28:01.131866 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:28:01.131907 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:28:04.465341 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:28:04.465385 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:28:07.800906 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:28:07.800957 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:28:11.134171 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:28:11.134332 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:28:14.466103 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:28:14.466229 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:30:27.291268 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:30:27.291316 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:30:30.624766 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:30:30.624813 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:30:31.021985 7 controller.go:1441] Error getting SSL certificate "auth-system/keycloak.199-19-213-193.nip.io-tls": local SSL certificate auth-system/keycloak.199-19-213-193.nip.io-tls was not found. Using default certificate W0721 05:30:31.022104 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:30:31.022130 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0721 05:30:31.024097 7 admission.go:149] processed ingress via admission controller {testedIngressLength:1 testedIngressTime:0.003s renderingIngressLength:1 renderingIngressTime:0s admissionTime:0.003s testedConfigurationSize:14.7kB} I0721 05:30:31.024137 7 main.go:107] "successfully validated configuration, accepting" ingress="auth-system/keycloak" I0721 05:30:31.031055 7 store.go:440] "Found valid IngressClass" ingress="auth-system/keycloak" ingressclass="atmosphere" I0721 05:30:31.031348 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"auth-system", Name:"keycloak", UID:"200b55c2-33a8-44de-9ce4-c76fe899dc86", APIVersion:"networking.k8s.io/v1", ResourceVersion:"3142", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0721 05:30:31.031374 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "auth-system/keycloak.199-19-213-193.nip.io-tls" in local store I0721 05:30:31.667639 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="auth-system/keycloak.199-19-213-193.nip.io-tls" I0721 05:30:31.668796 7 backend_ssl.go:67] "Adding secret to local store" name="auth-system/keycloak.199-19-213-193.nip.io-tls" W0721 05:30:33.959046 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:30:33.959104 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0721 05:30:33.959176 7 controller.go:196] "Configuration changes detected, backend reload required" I0721 05:30:34.003678 7 controller.go:216] "Backend successfully reloaded" I0721 05:30:34.003997 7 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-vw7wm", UID:"a91d7e05-6640-418e-825c-ea1b85e5c9c4", APIVersion:"v1", ResourceVersion:"1410", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration W0721 05:30:37.292371 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:30:37.292407 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 199.19.213.193 - - [21/Jul/2026:05:30:39 +0000] "GET /realms/master HTTP/1.1" 200 624 "-" "ansible-httpget" 144 0.010 [auth-system-keycloak-80] [] 10.0.0.229:8080 624 0.010 200 a0e82f56b1e0517635af6edd4ae24e81 199.19.213.193 - - [21/Jul/2026:05:30:41 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1885 "-" "Ansible" 332 1.686 [auth-system-keycloak-80] [] 10.0.0.229:8080 1885 1.685 200 dacdc1ed6627bf48b00872444e9aed3b 199.19.213.193 - - [21/Jul/2026:05:30:42 +0000] "GET /admin/realms/atmosphere HTTP/1.1" 404 28 "-" "Ansible" 1197 0.179 [auth-system-keycloak-80] [] 10.0.0.229:8080 28 0.179 404 cd1b7935bb34606a58d6360586b37ac4 199.19.213.193 - - [21/Jul/2026:05:30:47 +0000] "POST /admin/realms HTTP/1.1" 201 0 "-" "Ansible" 1296 5.083 [auth-system-keycloak-80] [] 10.0.0.229:8080 0 5.082 201 91e1307bfa5925b3053833e3a7d90313 199.19.213.193 - - [21/Jul/2026:05:30:47 +0000] "GET /admin/realms/atmosphere HTTP/1.1" 200 4177 "-" "Ansible" 1197 0.614 [auth-system-keycloak-80] [] 10.0.0.229:8080 4177 0.614 200 83e21bd2819e73a57a398f828d7be2ac 199.19.213.193 - - [21/Jul/2026:05:30:48 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1885 "-" "Ansible" 332 0.062 [auth-system-keycloak-80] [] 10.0.0.229:8080 1885 0.060 200 d2fb1db8ced54026fa4aac44c6b18870 199.19.213.193 - - [21/Jul/2026:05:30:48 +0000] "GET /admin/realms/atmosphere/client-scopes HTTP/1.1" 200 12500 "-" "Ansible" 1211 0.068 [auth-system-keycloak-80] [] 10.0.0.229:8080 12500 0.068 200 ce83a7d1aae95da228da8077defd6be1 199.19.213.193 - - [21/Jul/2026:05:30:48 +0000] "GET /admin/realms/atmosphere/client-scopes/46c872ab-7970-40fb-9f23-6ca8fa4943ea HTTP/1.1" 200 1278 "-" "Ansible" 1248 0.015 [auth-system-keycloak-80] [] 10.0.0.229:8080 1278 0.015 200 b705cb6c849b5232e742f8cc49d1fea7 199.19.213.193 - - [21/Jul/2026:05:30:48 +0000] "PUT /admin/realms/atmosphere/client-scopes/46c872ab-7970-40fb-9f23-6ca8fa4943ea HTTP/1.1" 204 0 "-" "Ansible" 1848 0.063 [auth-system-keycloak-80] [] 10.0.0.229:8080 0 0.062 204 9b05ed51278f79cc6c1d4a0d06120886 199.19.213.193 - - [21/Jul/2026:05:30:48 +0000] "GET /admin/realms/atmosphere/client-scopes/46c872ab-7970-40fb-9f23-6ca8fa4943ea/protocol-mappers/models HTTP/1.1" 200 963 "-" "Ansible" 1272 0.086 [auth-system-keycloak-80] [] 10.0.0.229:8080 963 0.086 200 62a1a15c4f0a0afc7b28515c4d0fd7a8 199.19.213.193 - - [21/Jul/2026:05:30:48 +0000] "GET /admin/realms/atmosphere/client-scopes/46c872ab-7970-40fb-9f23-6ca8fa4943ea/protocol-mappers/models/12ad3312-fb95-4a06-81e1-916e90c30c3c HTTP/1.1" 200 366 "-" "Ansible" 1309 0.010 [auth-system-keycloak-80] [] 10.0.0.229:8080 366 0.010 200 7350726aef0cf23fe4fb71bd6727d8b9 199.19.213.193 - - [21/Jul/2026:05:30:48 +0000] "PUT /admin/realms/atmosphere/client-scopes/46c872ab-7970-40fb-9f23-6ca8fa4943ea/protocol-mappers/models/12ad3312-fb95-4a06-81e1-916e90c30c3c HTTP/1.1" 204 0 "-" "Ansible" 1622 0.078 [auth-system-keycloak-80] [] 10.0.0.229:8080 0 0.078 204 080c2f11d42fb15f93ec4487032f27ab 199.19.213.193 - - [21/Jul/2026:05:30:48 +0000] "GET /admin/realms/atmosphere/client-scopes/46c872ab-7970-40fb-9f23-6ca8fa4943ea HTTP/1.1" 200 1218 "-" "Ansible" 1248 0.080 [auth-system-keycloak-80] [] 10.0.0.229:8080 1218 0.080 200 aff3d3661a89b32a255c51ca0241104f E0721 05:30:57.375050 7 leaderelection.go:429] Failed to update lock optimistically: etcdserver: request timed out, falling back to slow path E0721 05:31:05.371377 7 leaderelection.go:436] error retrieving resource lock ingress-nginx/ingress-nginx-leader: Get "https://10.96.0.1:443/apis/coordination.k8s.io/v1/namespaces/ingress-nginx/leases/ingress-nginx-leader": context deadline exceeded I0721 05:31:05.371431 7 leaderelection.go:297] failed to renew lease ingress-nginx/ingress-nginx-leader: context deadline exceeded E0721 05:31:05.371610 7 status.go:104] "error running poll" err="timed out waiting for the condition" I0721 05:31:05.371685 7 leaderelection.go:257] attempting to acquire leader lease ingress-nginx/ingress-nginx-leader... E0721 05:31:11.456417 7 leaderelection.go:429] Failed to update lock optimistically: Operation cannot be fulfilled on leases.coordination.k8s.io "ingress-nginx-leader": the object has been modified; please apply your changes to the latest version and try again, falling back to slow path I0721 05:31:12.118890 7 leaderelection.go:271] successfully acquired lease ingress-nginx/ingress-nginx-leader I0721 05:31:12.332853 7 status.go:304] "updating Ingress status" namespace="auth-system" ingress="keycloak" currentValue=null newValue=[{"ip":"10.107.15.202"}] I0721 05:31:12.658637 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"auth-system", Name:"keycloak", UID:"200b55c2-33a8-44de-9ce4-c76fe899dc86", APIVersion:"networking.k8s.io/v1", ResourceVersion:"3440", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0721 05:31:12.665024 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:31:12.665083 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:31:35.384378 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:31:35.384408 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:32:04.538758 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0721 05:32:04.538786 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store