------------------------------------------------------------------------------- NGINX Ingress controller Release: v1.12.1 Build: 51c2b819690bbf1709b844dbf321a9acf6eda5a7 W0622 01:49:59.537225 7 client_config.go:667] Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work. Repository: https://github.com/kubernetes/ingress-nginx I0622 01:49:59.537377 7 main.go:205] "Creating API client" host="https://10.96.0.1:443" nginx version: nginx/1.25.5 I0622 01:49:59.545679 7 main.go:248] "Running in Kubernetes cluster" major="1" minor="28" git="v1.28.13" state="clean" commit="024ab2a13ff700705efd3ab63a0e27c38c462070" platform="linux/amd64" ------------------------------------------------------------------------------- I0622 01:49:59.597976 7 main.go:83] "Valid default backend" service="ingress-nginx/ingress-nginx-defaultbackend" I0622 01:49:59.964856 7 main.go:101] "SSL fake certificate created" file="/etc/ingress-controller/ssl/default-fake-certificate.pem" I0622 01:50:00.372195 7 ssl.go:535] "loading tls certificate" path="/usr/local/certificates/cert" key="/usr/local/certificates/key" I0622 01:50:00.412357 7 nginx.go:271] "Starting NGINX Ingress controller" I0622 01:50:00.418755 7 event.go:377] Event(v1.ObjectReference{Kind:"ConfigMap", Namespace:"ingress-nginx", Name:"ingress-nginx-controller", UID:"230d09eb-6b9c-4add-a9dc-469d200d1a56", APIVersion:"v1", ResourceVersion:"1424", FieldPath:""}): type: 'Normal' reason: 'CREATE' ConfigMap ingress-nginx/ingress-nginx-controller I0622 01:50:00.422297 7 event.go:377] Event(v1.ObjectReference{Kind:"ConfigMap", Namespace:"ingress-nginx", Name:"ingress-nginx-tcp", UID:"a6e62b76-77e5-4518-9aac-40316b73b94a", APIVersion:"v1", ResourceVersion:"1423", FieldPath:""}): type: 'Normal' reason: 'CREATE' ConfigMap ingress-nginx/ingress-nginx-tcp I0622 01:50:00.422812 7 event.go:377] Event(v1.ObjectReference{Kind:"ConfigMap", Namespace:"ingress-nginx", Name:"ingress-nginx-udp", UID:"2cbc2970-f55d-45ee-b1ff-629f2851203c", APIVersion:"v1", ResourceVersion:"1422", FieldPath:""}): type: 'Normal' reason: 'CREATE' ConfigMap ingress-nginx/ingress-nginx-udp I0622 01:50:01.614466 7 nginx.go:317] "Starting NGINX process" I0622 01:50:01.614523 7 leaderelection.go:257] attempting to acquire leader lease ingress-nginx/ingress-nginx-leader... I0622 01:50:01.614781 7 nginx.go:337] "Starting validation webhook" address=":7443" certPath="/usr/local/certificates/cert" keyPath="/usr/local/certificates/key" W0622 01:50:01.614945 7 controller.go:603] Service "ingress-nginx/ingress-nginx-defaultbackend" does not have any active Endpoint W0622 01:50:01.614990 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:01.615003 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0622 01:50:01.615043 7 controller.go:196] "Configuration changes detected, backend reload required" I0622 01:50:01.758382 7 controller.go:216] "Backend successfully reloaded" I0622 01:50:01.758518 7 controller.go:227] "Initial sync, sleeping for 1 second" I0622 01:50:01.758599 7 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-r9rwh", UID:"0c9749e2-a13d-4739-b0a7-e745fcd477a9", APIVersion:"v1", ResourceVersion:"1467", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration I0622 01:50:02.391371 7 leaderelection.go:271] successfully acquired lease ingress-nginx/ingress-nginx-leader I0622 01:50:02.391698 7 status.go:85] "New leader elected" identity="ingress-nginx-controller-r9rwh" W0622 01:50:05.403208 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:05.403241 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:08.737125 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:08.737168 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:12.070988 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:12.071025 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:15.403641 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:15.403683 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:18.737156 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:18.737202 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:23.974281 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:23.974321 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:27.309223 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:27.309255 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:30.641788 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:30.641829 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:33.975134 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:33.975179 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:38.709912 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:38.709939 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:42.044465 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:50:42.044514 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:51:35.929753 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:51:35.929799 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:51:39.264722 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:51:39.265007 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:51:44.983483 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:51:44.983523 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:51:48.317264 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:51:48.317308 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:51:54.742972 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:51:54.743009 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:51:58.076443 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:51:58.076475 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:52:01.410439 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:52:01.410523 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:52:04.743607 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:52:04.743636 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:52:08.078884 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:52:08.078928 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:52:11.410337 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:52:11.410367 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:52:14.744116 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:52:14.744156 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:52:18.076679 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:52:18.076725 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:54:26.449273 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:54:26.449316 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:54:29.224156 7 controller.go:1441] Error getting SSL certificate "auth-system/keycloak.199-204-45-144.nip.io-tls": local SSL certificate auth-system/keycloak.199-204-45-144.nip.io-tls was not found. Using default certificate W0622 01:54:29.224220 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:54:29.224242 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0622 01:54:29.227408 7 admission.go:149] processed ingress via admission controller {testedIngressLength:1 testedIngressTime:0.004s renderingIngressLength:1 renderingIngressTime:0s admissionTime:0.004s testedConfigurationSize:14.7kB} I0622 01:54:29.227448 7 main.go:107] "successfully validated configuration, accepting" ingress="auth-system/keycloak" I0622 01:54:29.234874 7 store.go:440] "Found valid IngressClass" ingress="auth-system/keycloak" ingressclass="atmosphere" W0622 01:54:29.235456 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "auth-system/keycloak.199-204-45-144.nip.io-tls" in local store I0622 01:54:29.235575 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"auth-system", Name:"keycloak", UID:"6ebdefc2-0816-4cfd-a9ee-dfff60859956", APIVersion:"networking.k8s.io/v1", ResourceVersion:"3391", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync I0622 01:54:29.590034 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="auth-system/keycloak.199-204-45-144.nip.io-tls" I0622 01:54:29.591383 7 backend_ssl.go:67] "Adding secret to local store" name="auth-system/keycloak.199-204-45-144.nip.io-tls" W0622 01:54:29.783667 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:54:29.783696 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0622 01:54:29.783737 7 controller.go:196] "Configuration changes detected, backend reload required" I0622 01:54:29.832823 7 controller.go:216] "Backend successfully reloaded" I0622 01:54:29.833059 7 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-r9rwh", UID:"0c9749e2-a13d-4739-b0a7-e745fcd477a9", APIVersion:"v1", ResourceVersion:"1467", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration W0622 01:54:33.121045 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:54:33.121078 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 199.204.45.144 - - [22/Jun/2026:01:54:34 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 2.088 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 2.085 200 d7930329be9c283771276309e608cd00 199.204.45.144 - - [22/Jun/2026:01:54:35 +0000] "GET /admin/realms/atmosphere HTTP/1.1" 404 116 "-" "Ansible" 1197 0.175 [auth-system-keycloak-80] [] 10.0.0.32:8080 116 0.176 404 467d93aa06b1861e7397507e7d6af03f 199.204.45.144 - - [22/Jun/2026:01:54:39 +0000] "POST /admin/realms HTTP/1.1" 201 0 "-" "Ansible" 1296 4.307 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 4.306 201 dec4adfc787d75ba50cd91b82b4f3c40 199.204.45.144 - - [22/Jun/2026:01:54:40 +0000] "GET /admin/realms/atmosphere HTTP/1.1" 200 4246 "-" "Ansible" 1197 0.686 [auth-system-keycloak-80] [] 10.0.0.32:8080 4246 0.686 200 c4ec04847e9609476ea3febe5f90e878 199.204.45.144 - - [22/Jun/2026:01:54:41 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.533 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 0.533 200 541bf2829382e69e7da03dceb8961ff0 199.204.45.144 - - [22/Jun/2026:01:54:41 +0000] "GET /admin/realms/atmosphere/client-scopes HTTP/1.1" 200 12492 "-" "Ansible" 1211 0.105 [auth-system-keycloak-80] [] 10.0.0.32:8080 12500 0.105 200 52dcc22aaca6d00b3973dcfabb9c80f1 199.204.45.144 - - [22/Jun/2026:01:54:41 +0000] "GET /admin/realms/atmosphere/client-scopes/2bdee88b-7be7-4950-806a-8d1be2376b9b HTTP/1.1" 200 1278 "-" "Ansible" 1248 0.015 [auth-system-keycloak-80] [] 10.0.0.32:8080 1278 0.015 200 5e38a185d9208bbdff9fda30d5e81196 199.204.45.144 - - [22/Jun/2026:01:54:41 +0000] "PUT /admin/realms/atmosphere/client-scopes/2bdee88b-7be7-4950-806a-8d1be2376b9b HTTP/1.1" 204 0 "-" "Ansible" 1848 0.074 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.074 204 eed963d69b5791dc1c15301b4ede8a23 199.204.45.144 - - [22/Jun/2026:01:54:41 +0000] "GET /admin/realms/atmosphere/client-scopes/2bdee88b-7be7-4950-806a-8d1be2376b9b/protocol-mappers/models HTTP/1.1" 200 963 "-" "Ansible" 1272 0.083 [auth-system-keycloak-80] [] 10.0.0.32:8080 963 0.082 200 2c1a054d45084b027a70365ce6c0d2aa 199.204.45.144 - - [22/Jun/2026:01:54:41 +0000] "GET /admin/realms/atmosphere/client-scopes/2bdee88b-7be7-4950-806a-8d1be2376b9b/protocol-mappers/models/0417530c-4d2e-4429-b6f9-c2116a754685 HTTP/1.1" 200 366 "-" "Ansible" 1309 0.011 [auth-system-keycloak-80] [] 10.0.0.32:8080 366 0.012 200 211da5e06825d7e850f01d6d864e694d 199.204.45.144 - - [22/Jun/2026:01:54:41 +0000] "PUT /admin/realms/atmosphere/client-scopes/2bdee88b-7be7-4950-806a-8d1be2376b9b/protocol-mappers/models/0417530c-4d2e-4429-b6f9-c2116a754685 HTTP/1.1" 204 0 "-" "Ansible" 1622 0.114 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.113 204 256531c17c3f58bd259313a2a0da0738 199.204.45.144 - - [22/Jun/2026:01:54:41 +0000] "GET /admin/realms/atmosphere/client-scopes/2bdee88b-7be7-4950-806a-8d1be2376b9b HTTP/1.1" 200 1218 "-" "Ansible" 1248 0.066 [auth-system-keycloak-80] [] 10.0.0.32:8080 1218 0.066 200 d11405ad5a03fd1dac90cf3c46ee208c I0622 01:55:02.400910 7 status.go:304] "updating Ingress status" namespace="auth-system" ingress="keycloak" currentValue=null newValue=[{"ip":"10.111.168.105"}] I0622 01:55:02.404700 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"auth-system", Name:"keycloak", UID:"6ebdefc2-0816-4cfd-a9ee-dfff60859956", APIVersion:"networking.k8s.io/v1", ResourceVersion:"3654", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0622 01:55:02.406087 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:55:02.406124 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 199.204.45.144 - - [22/Jun/2026:01:55:03 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.481 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 0.480 200 1508b12634fc035f068729e844ba8850 199.204.45.144 - - [22/Jun/2026:01:55:03 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 2 "-" "Ansible" 1227 0.051 [auth-system-keycloak-80] [] 10.0.0.32:8080 2 0.050 200 79aa1088454932870d056e0cda13ed03 199.204.45.144 - - [22/Jun/2026:01:55:03 +0000] "POST /admin/realms/atmosphere/clients HTTP/1.1" 201 0 "-" "Ansible" 1657 0.201 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.199 201 5b0e402c01e5380ce996fe72303f8451 199.204.45.144 - - [22/Jun/2026:01:55:03 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 1261 "-" "Ansible" 1227 0.081 [auth-system-keycloak-80] [] 10.0.0.32:8080 1261 0.081 200 9f3ea1408b42d71fede4da5f9e986596 199.204.45.144 - - [22/Jun/2026:01:55:04 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 1.162 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 1.161 200 e5b217f43edcae7b9c51237061664132 199.204.45.144 - - [22/Jun/2026:01:55:05 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 2 "-" "Ansible" 1222 0.012 [auth-system-keycloak-80] [] 10.0.0.32:8080 2 0.012 200 89428fe981a8a327f7a1841f746d4406 199.204.45.144 - - [22/Jun/2026:01:55:05 +0000] "POST /admin/realms/atmosphere/clients HTTP/1.1" 201 0 "-" "Ansible" 1688 0.059 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.057 201 ae826e299285bdfe236e4ebd747e85ed 199.204.45.144 - - [22/Jun/2026:01:55:05 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.021 [auth-system-keycloak-80] [] 10.0.0.32:8080 1286 0.021 200 e593a0a0db0c7b704ba3a5cacc896ca1 199.204.45.144 - - [22/Jun/2026:01:55:06 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.501 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 0.500 200 0d60f5f274f22c0b8e217ced30c1ed0b 199.204.45.144 - - [22/Jun/2026:01:55:06 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 2 "-" "Ansible" 1225 0.009 [auth-system-keycloak-80] [] 10.0.0.32:8080 2 0.009 200 5f452206294f01a77c9e5564193822f3 199.204.45.144 - - [22/Jun/2026:01:55:06 +0000] "POST /admin/realms/atmosphere/clients HTTP/1.1" 201 0 "-" "Ansible" 1649 0.188 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.188 201 92335d95154723af879eb988669fa66e 199.204.45.144 - - [22/Jun/2026:01:55:06 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 1251 "-" "Ansible" 1225 0.023 [auth-system-keycloak-80] [] 10.0.0.32:8080 1251 0.024 200 e629363d0b57fa4fd9e1a12fd6d7a356 199.204.45.144 - - [22/Jun/2026:01:55:07 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.502 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 0.502 200 6f3d1f9fe4200cdb0636b62b9689d596 199.204.45.144 - - [22/Jun/2026:01:55:07 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 1261 "-" "Ansible" 1227 0.008 [auth-system-keycloak-80] [] 10.0.0.32:8080 1261 0.008 200 88c71667919d11ae16e752a382e675a3 199.204.45.144 - - [22/Jun/2026:01:55:07 +0000] "GET /admin/realms/atmosphere/clients/f87425e3-1a68-4c92-8e6a-79e836ce6573/roles/member HTTP/1.1" 404 119 "-" "Ansible" 1255 0.014 [auth-system-keycloak-80] [] 10.0.0.32:8080 119 0.014 404 5263aef9d7fd26e50e386519b853a391 199.204.45.144 - - [22/Jun/2026:01:55:07 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 1261 "-" "Ansible" 1227 0.041 [auth-system-keycloak-80] [] 10.0.0.32:8080 1261 0.041 200 3fa728cf3567b52daa8a2af3cdbcae38 199.204.45.144 - - [22/Jun/2026:01:55:07 +0000] "POST /admin/realms/atmosphere/clients/f87425e3-1a68-4c92-8e6a-79e836ce6573/roles HTTP/1.1" 201 0 "-" "Ansible" 1350 0.086 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.085 201 c3b836c7566e959242906d2631cddcb1 199.204.45.144 - - [22/Jun/2026:01:55:07 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 1261 "-" "Ansible" 1227 0.009 [auth-system-keycloak-80] [] 10.0.0.32:8080 1261 0.009 200 18771a75e77d9dc7534d5fd079b7adcb 199.204.45.144 - - [22/Jun/2026:01:55:08 +0000] "GET /admin/realms/atmosphere/clients/f87425e3-1a68-4c92-8e6a-79e836ce6573/roles/member HTTP/1.1" 200 166 "-" "Ansible" 1255 0.077 [auth-system-keycloak-80] [] 10.0.0.32:8080 166 0.077 200 c9ba17e3f5782d3d9325d2692a1bfc8b 199.204.45.144 - - [22/Jun/2026:01:55:08 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.490 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 0.489 200 b93716f1c385aedcf88ca0014100f21e 199.204.45.144 - - [22/Jun/2026:01:55:08 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.012 [auth-system-keycloak-80] [] 10.0.0.32:8080 1286 0.012 200 33ea28ed5dd5189c9e54e1c781558072 199.204.45.144 - - [22/Jun/2026:01:55:08 +0000] "GET /admin/realms/atmosphere/clients/13af98d9-da08-478e-a610-31c5ebff93c9/roles/admin HTTP/1.1" 404 119 "-" "Ansible" 1254 0.011 [auth-system-keycloak-80] [] 10.0.0.32:8080 119 0.010 404 b4678f1f0053ea5f14c144a3813fea6f 199.204.45.144 - - [22/Jun/2026:01:55:08 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.027 [auth-system-keycloak-80] [] 10.0.0.32:8080 1286 0.026 200 9dd925fc4ec8a9596096dbc2150953ee 199.204.45.144 - - [22/Jun/2026:01:55:09 +0000] "POST /admin/realms/atmosphere/clients/13af98d9-da08-478e-a610-31c5ebff93c9/roles HTTP/1.1" 201 0 "-" "Ansible" 1349 0.088 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.087 201 968805e11df8dd27221a1d7c8cc3ad27 199.204.45.144 - - [22/Jun/2026:01:55:09 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.008 [auth-system-keycloak-80] [] 10.0.0.32:8080 1286 0.009 200 8feda5d44c4ad3a25d38597fb6689a2e 199.204.45.144 - - [22/Jun/2026:01:55:09 +0000] "GET /admin/realms/atmosphere/clients/13af98d9-da08-478e-a610-31c5ebff93c9/roles/admin HTTP/1.1" 200 165 "-" "Ansible" 1254 0.015 [auth-system-keycloak-80] [] 10.0.0.32:8080 165 0.015 200 9a90adb91c2b8efe709f9d705e6fc207 199.204.45.144 - - [22/Jun/2026:01:55:09 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.488 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 0.487 200 685dcc1d2b136af0a0312eaf44894f53 199.204.45.144 - - [22/Jun/2026:01:55:09 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.070 [auth-system-keycloak-80] [] 10.0.0.32:8080 1286 0.070 200 25cd83f71e362966c255a32309ea1afd 199.204.45.144 - - [22/Jun/2026:01:55:09 +0000] "GET /admin/realms/atmosphere/clients/13af98d9-da08-478e-a610-31c5ebff93c9/roles/editor HTTP/1.1" 404 119 "-" "Ansible" 1255 0.012 [auth-system-keycloak-80] [] 10.0.0.32:8080 119 0.012 404 36e69e4e58a0debac4ab3a67efb71564 199.204.45.144 - - [22/Jun/2026:01:55:09 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.007 [auth-system-keycloak-80] [] 10.0.0.32:8080 1286 0.007 200 dfd083fde7377ef30b2f07f2c1b74456 199.204.45.144 - - [22/Jun/2026:01:55:10 +0000] "POST /admin/realms/atmosphere/clients/13af98d9-da08-478e-a610-31c5ebff93c9/roles HTTP/1.1" 201 0 "-" "Ansible" 1350 0.060 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.059 201 5bfeb1264344d02d9954ff4cc22975d2 199.204.45.144 - - [22/Jun/2026:01:55:10 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.006 [auth-system-keycloak-80] [] 10.0.0.32:8080 1286 0.006 200 d37f06d281fc7041efb1a38f2dde95aa 199.204.45.144 - - [22/Jun/2026:01:55:10 +0000] "GET /admin/realms/atmosphere/clients/13af98d9-da08-478e-a610-31c5ebff93c9/roles/editor HTTP/1.1" 200 166 "-" "Ansible" 1255 0.016 [auth-system-keycloak-80] [] 10.0.0.32:8080 166 0.016 200 dd88e3dd1d39660702bf2fbf58c9cb72 199.204.45.144 - - [22/Jun/2026:01:55:10 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.535 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 0.536 200 44165e9bbc7130ed6e10c261952680a2 199.204.45.144 - - [22/Jun/2026:01:55:10 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.015 [auth-system-keycloak-80] [] 10.0.0.32:8080 1286 0.016 200 7f7682bf23fe0ae335471316517190a9 199.204.45.144 - - [22/Jun/2026:01:55:10 +0000] "GET /admin/realms/atmosphere/clients/13af98d9-da08-478e-a610-31c5ebff93c9/roles/viewer HTTP/1.1" 404 119 "-" "Ansible" 1255 0.011 [auth-system-keycloak-80] [] 10.0.0.32:8080 119 0.011 404 ace02e538e036247cd99ea49b69d143e 199.204.45.144 - - [22/Jun/2026:01:55:11 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.040 [auth-system-keycloak-80] [] 10.0.0.32:8080 1286 0.039 200 33a8299b41ebe8b680e99ccd4b7b045f 199.204.45.144 - - [22/Jun/2026:01:55:11 +0000] "POST /admin/realms/atmosphere/clients/13af98d9-da08-478e-a610-31c5ebff93c9/roles HTTP/1.1" 201 0 "-" "Ansible" 1350 0.016 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.016 201 009f6797dc309d70c409e2a1f82975f4 199.204.45.144 - - [22/Jun/2026:01:55:11 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.007 [auth-system-keycloak-80] [] 10.0.0.32:8080 1286 0.007 200 0141fa2901f73bb7ec8406084f2b8b75 199.204.45.144 - - [22/Jun/2026:01:55:11 +0000] "GET /admin/realms/atmosphere/clients/13af98d9-da08-478e-a610-31c5ebff93c9/roles/viewer HTTP/1.1" 200 166 "-" "Ansible" 1255 0.053 [auth-system-keycloak-80] [] 10.0.0.32:8080 166 0.053 200 93595c1c7188625cf27c1fe367ce8c00 199.204.45.144 - - [22/Jun/2026:01:55:11 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.561 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 0.561 200 93fedda7347a53e8be1ceb6910b607d5 199.204.45.144 - - [22/Jun/2026:01:55:12 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 1251 "-" "Ansible" 1225 0.036 [auth-system-keycloak-80] [] 10.0.0.32:8080 1251 0.036 200 d4cb053333e959caf8801891ef6a4d82 199.204.45.144 - - [22/Jun/2026:01:55:12 +0000] "GET /admin/realms/atmosphere/clients/9298a99d-883f-49ea-9772-35f80a648712/roles/member HTTP/1.1" 404 119 "-" "Ansible" 1255 0.009 [auth-system-keycloak-80] [] 10.0.0.32:8080 119 0.009 404 7f3b9f81b8fccba35413fbeedb4a5d04 199.204.45.144 - - [22/Jun/2026:01:55:12 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 1251 "-" "Ansible" 1225 0.007 [auth-system-keycloak-80] [] 10.0.0.32:8080 1251 0.008 200 c73fbf7e1f33f35db840ed359b9507cc 199.204.45.144 - - [22/Jun/2026:01:55:12 +0000] "POST /admin/realms/atmosphere/clients/9298a99d-883f-49ea-9772-35f80a648712/roles HTTP/1.1" 201 0 "-" "Ansible" 1350 0.062 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.061 201 ffee8a7f562709d4a59ee18211b13921 199.204.45.144 - - [22/Jun/2026:01:55:12 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 1251 "-" "Ansible" 1225 0.007 [auth-system-keycloak-80] [] 10.0.0.32:8080 1251 0.007 200 1ecc6269c15cd933ce4f0a7352bf9cad 199.204.45.144 - - [22/Jun/2026:01:55:12 +0000] "GET /admin/realms/atmosphere/clients/9298a99d-883f-49ea-9772-35f80a648712/roles/member HTTP/1.1" 200 166 "-" "Ansible" 1255 0.013 [auth-system-keycloak-80] [] 10.0.0.32:8080 166 0.013 200 7210d54ccbfab0516ba5c32d8a3aafa2 W0622 01:56:30.479204 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:30.479239 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:30.554947 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:56:30.555014 7 controller.go:1441] Error getting SSL certificate "monitoring/grafana-tls": local SSL certificate monitoring/grafana-tls was not found. Using default certificate W0622 01:56:30.555145 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:30.555165 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:30.556028 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:56:30.556087 7 controller.go:1441] Error getting SSL certificate "monitoring/prometheus-tls": local SSL certificate monitoring/prometheus-tls was not found. Using default certificate W0622 01:56:30.556115 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:30.556130 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:30.556504 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:56:30.556584 7 controller.go:1441] Error getting SSL certificate "monitoring/alertmanager-tls": local SSL certificate monitoring/alertmanager-tls was not found. Using default certificate W0622 01:56:30.556635 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:30.556652 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0622 01:56:30.557188 7 admission.go:149] processed ingress via admission controller {testedIngressLength:2 testedIngressTime:0.003s renderingIngressLength:2 renderingIngressTime:0s admissionTime:0.003s testedConfigurationSize:18.2kB} I0622 01:56:30.557219 7 main.go:107] "successfully validated configuration, accepting" ingress="monitoring/kube-prometheus-stack-grafana" I0622 01:56:30.557515 7 admission.go:149] processed ingress via admission controller {testedIngressLength:2 testedIngressTime:0.002s renderingIngressLength:2 renderingIngressTime:0s admissionTime:0.002s testedConfigurationSize:18.2kB} I0622 01:56:30.557551 7 main.go:107] "successfully validated configuration, accepting" ingress="monitoring/kube-prometheus-stack-prometheus" I0622 01:56:30.558495 7 admission.go:149] processed ingress via admission controller {testedIngressLength:2 testedIngressTime:0.002s renderingIngressLength:2 renderingIngressTime:0.001s admissionTime:0.003s testedConfigurationSize:18.2kB} I0622 01:56:30.558529 7 main.go:107] "successfully validated configuration, accepting" ingress="monitoring/kube-prometheus-stack-alertmanager" I0622 01:56:30.565106 7 store.go:440] "Found valid IngressClass" ingress="monitoring/kube-prometheus-stack-prometheus" ingressclass="atmosphere" W0622 01:56:30.565695 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "monitoring/prometheus-tls" in local store I0622 01:56:30.565720 7 store.go:440] "Found valid IngressClass" ingress="monitoring/kube-prometheus-stack-grafana" ingressclass="atmosphere" W0622 01:56:30.566015 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "monitoring/grafana-tls" in local store I0622 01:56:30.566165 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-prometheus", UID:"bddb6e6c-edc7-49d3-ab83-12849781c500", APIVersion:"networking.k8s.io/v1", ResourceVersion:"4324", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync I0622 01:56:30.566200 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-grafana", UID:"ab7bb10a-ef23-41bc-938a-0c6c6489f687", APIVersion:"networking.k8s.io/v1", ResourceVersion:"4325", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync I0622 01:56:30.566336 7 store.go:440] "Found valid IngressClass" ingress="monitoring/kube-prometheus-stack-alertmanager" ingressclass="atmosphere" I0622 01:56:30.566555 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-alertmanager", UID:"d2792b84-c7df-4bf5-8472-a151a8183abb", APIVersion:"networking.k8s.io/v1", ResourceVersion:"4327", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0622 01:56:30.566895 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "monitoring/alertmanager-tls" in local store I0622 01:56:31.122518 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="monitoring/grafana-tls" I0622 01:56:31.123472 7 backend_ssl.go:67] "Adding secret to local store" name="monitoring/grafana-tls" I0622 01:56:32.020044 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="monitoring/alertmanager-tls" I0622 01:56:32.020881 7 backend_ssl.go:67] "Adding secret to local store" name="monitoring/alertmanager-tls" I0622 01:56:33.105568 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="monitoring/prometheus-tls" I0622 01:56:33.106449 7 backend_ssl.go:67] "Adding secret to local store" name="monitoring/prometheus-tls" W0622 01:56:33.812634 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:56:33.812691 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:56:33.812721 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:56:33.812796 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:33.812815 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0622 01:56:33.812925 7 controller.go:196] "Configuration changes detected, backend reload required" I0622 01:56:33.864353 7 controller.go:216] "Backend successfully reloaded" I0622 01:56:33.864545 7 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-r9rwh", UID:"0c9749e2-a13d-4739-b0a7-e745fcd477a9", APIVersion:"v1", ResourceVersion:"1467", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration W0622 01:56:37.147402 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:56:37.147499 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:56:37.147542 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:56:37.147688 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:37.147723 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:40.479649 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:56:40.479734 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:56:40.479806 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:56:40.480013 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:40.480046 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:43.813112 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:56:43.813165 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:56:43.813187 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:56:43.813306 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:43.813323 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:47.146717 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:56:47.146837 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:56:47.146978 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:56:47.147081 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:47.147112 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:50.898266 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:56:50.898324 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:56:50.898546 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:56:50.898732 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:50.898760 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:54.232651 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:56:54.232738 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:56:54.232775 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:56:54.233068 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:56:54.233097 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:01.218697 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:57:01.218917 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:57:01.218965 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:57:01.219190 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:01.219217 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0622 01:57:02.398070 7 status.go:304] "updating Ingress status" namespace="monitoring" ingress="kube-prometheus-stack-alertmanager" currentValue=null newValue=[{"ip":"10.111.168.105"}] I0622 01:57:02.398675 7 status.go:304] "updating Ingress status" namespace="monitoring" ingress="kube-prometheus-stack-prometheus" currentValue=null newValue=[{"ip":"10.111.168.105"}] I0622 01:57:02.398903 7 status.go:304] "updating Ingress status" namespace="monitoring" ingress="kube-prometheus-stack-grafana" currentValue=null newValue=[{"ip":"10.111.168.105"}] I0622 01:57:02.451015 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-alertmanager", UID:"d2792b84-c7df-4bf5-8472-a151a8183abb", APIVersion:"networking.k8s.io/v1", ResourceVersion:"4964", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync I0622 01:57:02.455048 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-prometheus", UID:"bddb6e6c-edc7-49d3-ab83-12849781c500", APIVersion:"networking.k8s.io/v1", ResourceVersion:"4965", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync I0622 01:57:02.463486 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-grafana", UID:"ab7bb10a-ef23-41bc-938a-0c6c6489f687", APIVersion:"networking.k8s.io/v1", ResourceVersion:"4966", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0622 01:57:04.553147 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:57:04.553214 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:57:04.553257 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:57:04.553460 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:04.553492 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:07.887112 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:57:07.887198 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:57:07.887239 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:57:07.887376 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:07.887419 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:11.219315 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:57:11.219355 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:57:11.219384 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:57:11.219468 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:11.219483 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:17.722584 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:57:17.722735 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0622 01:57:17.722831 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:57:17.723122 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:17.723220 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:21.340101 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:57:21.340547 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0622 01:57:21.340746 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:21.340811 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 10.0.0.124 - - [22/Jun/2026:01:57:21 +0000] "GET /realms/atmosphere/.well-known/openid-configuration HTTP/1.1" 200 6432 "-" "Go-http-client/1.1" 161 0.034 [auth-system-keycloak-80] [] 10.0.0.32:8080 6432 0.035 200 0776b2177c45240bcbbf8fa814fda00d 10.0.0.242 - - [22/Jun/2026:01:57:22 +0000] "GET /realms/atmosphere/.well-known/openid-configuration HTTP/1.1" 200 6432 "-" "Go-http-client/1.1" 161 0.008 [auth-system-keycloak-80] [] 10.0.0.32:8080 6432 0.008 200 4ff08098a8691fef1b5fe93b6dbf0014 W0622 01:57:24.673646 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:57:24.674051 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:24.674127 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:28.006810 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:57:28.007286 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:28.007324 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:33.935868 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:57:33.936060 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:33.936088 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:37.638762 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0622 01:57:37.638934 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:37.638977 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 199.204.45.144 - - [22/Jun/2026:01:57:39 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.488 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 0.487 200 b00b41ee73a204dfdeb91a72a4547348 199.204.45.144 - - [22/Jun/2026:01:57:39 +0000] "GET /admin/realms/atmosphere HTTP/1.1" 200 4246 "-" "Ansible" 1197 0.037 [auth-system-keycloak-80] [] 10.0.0.32:8080 4246 0.037 200 90d28d33a2ec57325f5f593cc9726eab 199.204.45.144 - - [22/Jun/2026:01:57:40 +0000] "PUT /admin/realms/atmosphere HTTP/1.1" 204 0 "-" "Ansible" 5709 0.582 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.581 204 e454d52db6e33ab27b4055489f32e4e5 199.204.45.144 - - [22/Jun/2026:01:57:40 +0000] "GET /admin/realms/atmosphere HTTP/1.1" 200 4384 "-" "Ansible" 1197 0.131 [auth-system-keycloak-80] [] 10.0.0.32:8080 4384 0.132 200 e0f24c73c2dd3a47bfa51ceda4f67106 W0622 01:57:40.972897 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:40.972932 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 199.204.45.144 - - [22/Jun/2026:01:57:41 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.479 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 0.478 200 108e9777fbea74fd0e0f631f28528be5 199.204.45.144 - - [22/Jun/2026:01:57:41 +0000] "GET /admin/realms/atmosphere/authentication/required-actions HTTP/1.1" 200 1682 "-" "Ansible" 1229 0.007 [auth-system-keycloak-80] [] 10.0.0.32:8080 1682 0.008 200 bd2493cf2e8460ab1543aaed9f4dbeb3 199.204.45.144 - - [22/Jun/2026:01:57:41 +0000] "GET /realms/atmosphere/.well-known/openid-configuration HTTP/1.1" 200 6432 "-" "ansible-httpget" 181 0.007 [auth-system-keycloak-80] [] 10.0.0.32:8080 6432 0.007 200 83cddec3830f26aa26421c0f5039fc89 199.204.45.144 - - [22/Jun/2026:01:57:43 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.490 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 0.489 200 d9eaed19e4799f9628493011112b7f83 199.204.45.144 - - [22/Jun/2026:01:57:43 +0000] "GET /admin/realms/atmosphere/clients?clientId=keystone HTTP/1.1" 200 2 "-" "Ansible" 1223 0.049 [auth-system-keycloak-80] [] 10.0.0.32:8080 2 0.050 200 4f80fb790510687c6570f44e200410b0 199.204.45.144 - - [22/Jun/2026:01:57:43 +0000] "POST /admin/realms/atmosphere/clients HTTP/1.1" 201 0 "-" "Ansible" 1628 0.193 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.192 201 a187d759a3e00e86ffb15d1a213ed58c 199.204.45.144 - - [22/Jun/2026:01:57:43 +0000] "GET /admin/realms/atmosphere/clients?clientId=keystone HTTP/1.1" 200 2246 "-" "Ansible" 1223 0.020 [auth-system-keycloak-80] [] 10.0.0.32:8080 2246 0.020 200 cef7e6d1b8044968919e00ece3bd07d8 199.204.45.144 - - [22/Jun/2026:01:57:44 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.526 [auth-system-keycloak-80] [] 10.0.0.32:8080 1928 0.525 200 fe40b24c20aaa5d758feca7c4744e012 199.204.45.144 - - [22/Jun/2026:01:57:44 +0000] "GET /admin/realms/atmosphere/clients?clientId=keystone HTTP/1.1" 200 2246 "-" "Ansible" 1223 0.035 [auth-system-keycloak-80] [] 10.0.0.32:8080 2246 0.035 200 6ab27a6b51add98a22643203eaea0949 199.204.45.144 - - [22/Jun/2026:01:57:44 +0000] "GET /admin/realms/atmosphere/clients/8f0b190d-3ee5-463b-80fd-0b773f2c1c58/service-account-user HTTP/1.1" 200 233 "-" "Ansible" 1263 0.016 [auth-system-keycloak-80] [] 10.0.0.32:8080 233 0.017 200 ac058b856d2a84ed430c7d040c453132 199.204.45.144 - - [22/Jun/2026:01:57:44 +0000] "GET /admin/realms/atmosphere/clients?clientId=realm-management HTTP/1.1" 200 831 "-" "Ansible" 1231 0.069 [auth-system-keycloak-80] [] 10.0.0.32:8080 831 0.069 200 8ae336c210acd06941bfb2f9d90f1cad 199.204.45.144 - - [22/Jun/2026:01:57:44 +0000] "GET /admin/realms/atmosphere/clients/72abe100-ac59-468b-a299-bbd60729237d/roles HTTP/1.1" 200 3758 "-" "Ansible" 1248 0.088 [auth-system-keycloak-80] [] 10.0.0.32:8080 3758 0.088 200 7e12d5f20a5fe3a4b24232b646142cd7 199.204.45.144 - - [22/Jun/2026:01:57:45 +0000] "GET /admin/realms/atmosphere/users/89aead1b-d7fc-4a76-b2d7-9308ba7f8a31/role-mappings/clients/72abe100-ac59-468b-a299-bbd60729237d/available HTTP/1.1" 200 3758 "-" "Ansible" 1309 0.175 [auth-system-keycloak-80] [] 10.0.0.32:8080 3758 0.175 200 0927e4f2e66cf5cf0523f84589a9f2b1 199.204.45.144 - - [22/Jun/2026:01:57:45 +0000] "GET /admin/realms/atmosphere/users/89aead1b-d7fc-4a76-b2d7-9308ba7f8a31/role-mappings/clients/72abe100-ac59-468b-a299-bbd60729237d/composite HTTP/1.1" 200 2 "-" "Ansible" 1309 0.019 [auth-system-keycloak-80] [] 10.0.0.32:8080 2 0.019 200 f1c563af0213e4cb3925e1567725f020 199.204.45.144 - - [22/Jun/2026:01:57:45 +0000] "POST /admin/realms/atmosphere/users/89aead1b-d7fc-4a76-b2d7-9308ba7f8a31/role-mappings/clients/72abe100-ac59-468b-a299-bbd60729237d HTTP/1.1" 204 0 "-" "Ansible" 1390 0.063 [auth-system-keycloak-80] [] 10.0.0.32:8080 0 0.062 204 46955376fac8bce8523cbe610ee2055a 199.204.45.144 - - [22/Jun/2026:01:57:45 +0000] "GET /admin/realms/atmosphere/users/89aead1b-d7fc-4a76-b2d7-9308ba7f8a31/role-mappings/clients/72abe100-ac59-468b-a299-bbd60729237d/composite HTTP/1.1" 200 576 "-" "Ansible" 1309 0.017 [auth-system-keycloak-80] [] 10.0.0.32:8080 576 0.017 200 001ae2a1b3b6de04d6e88925266276c7 W0622 01:57:48.080069 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:48.080166 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:51.414254 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:57:51.414345 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:59:53.777908 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 01:59:53.777936 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 02:00:27.246445 7 controller.go:1441] Error getting SSL certificate "openstack/keystone-api-certs": local SSL certificate openstack/keystone-api-certs was not found. Using default certificate W0622 02:00:27.246523 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 02:00:27.246816 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0622 02:00:27.250198 7 admission.go:149] processed ingress via admission controller {testedIngressLength:5 testedIngressTime:0.004s renderingIngressLength:5 renderingIngressTime:0.001s admissionTime:0.005s testedConfigurationSize:28.7kB} I0622 02:00:27.250243 7 main.go:107] "successfully validated configuration, accepting" ingress="openstack/identity" I0622 02:00:27.257814 7 store.go:440] "Found valid IngressClass" ingress="openstack/identity" ingressclass="atmosphere" I0622 02:00:27.258128 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"openstack", Name:"identity", UID:"5b556236-f802-44ba-b9ac-5fea893c6b64", APIVersion:"networking.k8s.io/v1", ResourceVersion:"6401", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0622 02:00:27.258352 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "openstack/keystone-api-certs" in local store W0622 02:00:27.258765 7 controller.go:1441] Error getting SSL certificate "openstack/keystone-api-certs": local SSL certificate openstack/keystone-api-certs was not found. Using default certificate W0622 02:00:27.258842 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 02:00:27.258864 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0622 02:00:27.258987 7 controller.go:196] "Configuration changes detected, backend reload required" I0622 02:00:27.315548 7 controller.go:216] "Backend successfully reloaded" I0622 02:00:27.315867 7 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-r9rwh", UID:"0c9749e2-a13d-4739-b0a7-e745fcd477a9", APIVersion:"v1", ResourceVersion:"1467", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration I0622 02:00:27.482218 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="openstack/keystone-api-certs" I0622 02:00:27.483964 7 backend_ssl.go:67] "Adding secret to local store" name="openstack/keystone-api-certs" W0622 02:00:30.593350 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 02:00:30.593402 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 199.204.45.144 - - [22/Jun/2026:02:00:30 +0000] "GET / HTTP/1.1" 300 281 "-" "ansible-httpget" 131 0.003 [openstack-keystone-api-5000] [] 10.0.0.215:5000 281 0.003 300 1a848ee623345591010b22dfe50fb6a7 E0622 02:00:36.998049 7 leaderelection.go:429] Failed to update lock optimistically: etcdserver: request timed out, falling back to slow path E0622 02:00:44.993534 7 leaderelection.go:436] error retrieving resource lock ingress-nginx/ingress-nginx-leader: Get "https://10.96.0.1:443/apis/coordination.k8s.io/v1/namespaces/ingress-nginx/leases/ingress-nginx-leader": context deadline exceeded I0622 02:00:44.993592 7 leaderelection.go:297] failed to renew lease ingress-nginx/ingress-nginx-leader: context deadline exceeded I0622 02:00:44.993699 7 leaderelection.go:257] attempting to acquire leader lease ingress-nginx/ingress-nginx-leader... E0622 02:00:44.994180 7 status.go:104] "error running poll" err="timed out waiting for the condition" E0622 02:00:51.998792 7 leaderelection.go:429] Failed to update lock optimistically: etcdserver: request timed out, falling back to slow path I0622 02:00:58.547169 7 leaderelection.go:271] successfully acquired lease ingress-nginx/ingress-nginx-leader I0622 02:00:59.902201 7 status.go:304] "updating Ingress status" namespace="openstack" ingress="identity" currentValue=null newValue=[{"ip":"10.111.168.105"}] I0622 02:01:00.907638 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"openstack", Name:"identity", UID:"5b556236-f802-44ba-b9ac-5fea893c6b64", APIVersion:"networking.k8s.io/v1", ResourceVersion:"6460", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0622 02:01:00.934799 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 02:01:00.934851 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 02:01:25.710199 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 02:01:25.710230 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 02:01:38.700321 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 02:01:38.700342 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 02:01:46.750452 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0622 02:01:46.750495 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store