------------------------------------------------------------------------------- NGINX Ingress controller Release: v1.12.1 Build: 51c2b819690bbf1709b844dbf321a9acf6eda5a7 Repository: https://github.com/kubernetes/ingress-nginx nginx version: nginx/1.25.5 ------------------------------------------------------------------------------- W0329 02:05:06.516580 7 client_config.go:667] Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work. I0329 02:05:06.516835 7 main.go:205] "Creating API client" host="https://10.96.0.1:443" I0329 02:05:06.523771 7 main.go:248] "Running in Kubernetes cluster" major="1" minor="28" git="v1.28.13" state="clean" commit="024ab2a13ff700705efd3ab63a0e27c38c462070" platform="linux/amd64" I0329 02:05:06.545408 7 main.go:83] "Valid default backend" service="ingress-nginx/ingress-nginx-defaultbackend" I0329 02:05:06.838901 7 main.go:101] "SSL fake certificate created" file="/etc/ingress-controller/ssl/default-fake-certificate.pem" I0329 02:05:06.861740 7 ssl.go:535] "loading tls certificate" path="/usr/local/certificates/cert" key="/usr/local/certificates/key" I0329 02:05:06.876546 7 nginx.go:271] "Starting NGINX Ingress controller" I0329 02:05:06.885128 7 event.go:377] Event(v1.ObjectReference{Kind:"ConfigMap", Namespace:"ingress-nginx", Name:"ingress-nginx-controller", UID:"a0725ca4-59c9-4b22-8047-4a52c11dc9eb", APIVersion:"v1", ResourceVersion:"1508", FieldPath:""}): type: 'Normal' reason: 'CREATE' ConfigMap ingress-nginx/ingress-nginx-controller I0329 02:05:06.889467 7 event.go:377] Event(v1.ObjectReference{Kind:"ConfigMap", Namespace:"ingress-nginx", Name:"ingress-nginx-tcp", UID:"2eb0f314-b2e1-44f2-ad4e-2247dd4ef5ee", APIVersion:"v1", ResourceVersion:"1507", FieldPath:""}): type: 'Normal' reason: 'CREATE' ConfigMap ingress-nginx/ingress-nginx-tcp I0329 02:05:06.889550 7 event.go:377] Event(v1.ObjectReference{Kind:"ConfigMap", Namespace:"ingress-nginx", Name:"ingress-nginx-udp", UID:"33e23d60-435b-4313-92c9-314fff79f347", APIVersion:"v1", ResourceVersion:"1506", FieldPath:""}): type: 'Normal' reason: 'CREATE' ConfigMap ingress-nginx/ingress-nginx-udp I0329 02:05:08.078000 7 nginx.go:317] "Starting NGINX process" I0329 02:05:08.078164 7 leaderelection.go:257] attempting to acquire leader lease ingress-nginx/ingress-nginx-leader... I0329 02:05:08.078586 7 nginx.go:337] "Starting validation webhook" address=":7443" certPath="/usr/local/certificates/cert" keyPath="/usr/local/certificates/key" W0329 02:05:08.078884 7 controller.go:603] Service "ingress-nginx/ingress-nginx-defaultbackend" does not have any active Endpoint W0329 02:05:08.078991 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:08.079023 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0329 02:05:08.079077 7 controller.go:196] "Configuration changes detected, backend reload required" I0329 02:05:08.087418 7 leaderelection.go:271] successfully acquired lease ingress-nginx/ingress-nginx-leader I0329 02:05:08.087506 7 status.go:85] "New leader elected" identity="ingress-nginx-controller-kbq2q" I0329 02:05:08.113162 7 controller.go:216] "Backend successfully reloaded" I0329 02:05:08.113355 7 controller.go:227] "Initial sync, sleeping for 1 second" I0329 02:05:08.113430 7 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-kbq2q", UID:"64a00c35-9bc1-4b5d-8b7a-ca800a9acc6a", APIVersion:"v1", ResourceVersion:"1552", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration W0329 02:05:11.909360 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:11.909389 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:15.243035 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:15.243106 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:18.576502 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:18.576548 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:21.912173 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:21.912218 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:25.243274 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:25.243335 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:31.992935 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:31.992972 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:35.326485 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:35.326536 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:38.660381 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:38.660430 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:41.994851 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:41.994905 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:45.329648 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:45.329705 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:48.660461 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:48.660494 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:56.719750 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:05:56.719798 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:06:00.097018 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:06:00.097055 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:18.308392 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:18.308449 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:21.644154 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:21.644211 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:33.681998 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:33.682031 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:37.015597 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:37.015654 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:40.349419 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:40.349465 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:43.682465 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:43.682530 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:47.015865 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:47.015901 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:50.349412 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:50.349524 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:53.684512 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:07:53.684684 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:08:00.206099 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:08:00.206157 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:08:09.260381 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:08:09.260421 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:08:12.594606 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:08:12.594663 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:10:50.819370 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:10:50.819426 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:10:53.649918 7 controller.go:1441] Error getting SSL certificate "auth-system/keycloak.199-204-45-116.nip.io-tls": local SSL certificate auth-system/keycloak.199-204-45-116.nip.io-tls was not found. Using default certificate W0329 02:10:53.649984 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:10:53.650004 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0329 02:10:53.652167 7 admission.go:149] processed ingress via admission controller {testedIngressLength:1 testedIngressTime:0.003s renderingIngressLength:1 renderingIngressTime:0s admissionTime:0.003s testedConfigurationSize:14.7kB} I0329 02:10:53.652206 7 main.go:107] "successfully validated configuration, accepting" ingress="auth-system/keycloak" I0329 02:10:53.660112 7 store.go:440] "Found valid IngressClass" ingress="auth-system/keycloak" ingressclass="atmosphere" I0329 02:10:53.660374 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"auth-system", Name:"keycloak", UID:"1f4f9923-82aa-4446-a6fb-c992e8ad6413", APIVersion:"networking.k8s.io/v1", ResourceVersion:"3916", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0329 02:10:53.660736 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "auth-system/keycloak.199-204-45-116.nip.io-tls" in local store I0329 02:10:53.948705 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="auth-system/keycloak.199-204-45-116.nip.io-tls" I0329 02:10:53.949635 7 backend_ssl.go:67] "Adding secret to local store" name="auth-system/keycloak.199-204-45-116.nip.io-tls" W0329 02:10:54.153553 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:10:54.153585 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0329 02:10:54.153628 7 controller.go:196] "Configuration changes detected, backend reload required" I0329 02:10:54.192331 7 controller.go:216] "Backend successfully reloaded" I0329 02:10:54.207356 7 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-kbq2q", UID:"64a00c35-9bc1-4b5d-8b7a-ca800a9acc6a", APIVersion:"v1", ResourceVersion:"1552", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration W0329 02:10:57.487043 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:10:57.487115 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 199.204.45.116 - - [29/Mar/2026:02:11:06 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 2.095 [auth-system-keycloak-80] [] 10.0.0.68:8080 1928 2.094 200 fa2a5ec3f9ef4569ea5920e5ce08274b 199.204.45.116 - - [29/Mar/2026:02:11:06 +0000] "GET /admin/realms/atmosphere HTTP/1.1" 404 116 "-" "Ansible" 1197 0.184 [auth-system-keycloak-80] [] 10.0.0.68:8080 116 0.184 404 0516c1a3fd2a8be03ef1d0d8dc9f039f I0329 02:11:08.094983 7 status.go:304] "updating Ingress status" namespace="auth-system" ingress="keycloak" currentValue=null newValue=[{"ip":"10.97.77.34"}] I0329 02:11:08.109173 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"auth-system", Name:"keycloak", UID:"1f4f9923-82aa-4446-a6fb-c992e8ad6413", APIVersion:"networking.k8s.io/v1", ResourceVersion:"4173", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0329 02:11:08.109835 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:11:08.109912 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 199.204.45.116 - - [29/Mar/2026:02:11:11 +0000] "POST /admin/realms HTTP/1.1" 201 0 "-" "Ansible" 1296 4.494 [auth-system-keycloak-80] [] 10.0.0.68:8080 0 4.493 201 daeb522d5f642171d35104319d3fedd0 199.204.45.116 - - [29/Mar/2026:02:11:11 +0000] "GET /admin/realms/atmosphere HTTP/1.1" 200 4246 "-" "Ansible" 1197 0.606 [auth-system-keycloak-80] [] 10.0.0.68:8080 4246 0.606 200 3e9ffe62402a7bd54f46cc59742e9286 199.204.45.116 - - [29/Mar/2026:02:11:12 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.478 [auth-system-keycloak-80] [] 10.0.0.68:8080 1928 0.476 200 f42bf9aed4cd8d887d40cf0e3491cfe3 199.204.45.116 - - [29/Mar/2026:02:11:12 +0000] "GET /admin/realms/atmosphere/client-scopes HTTP/1.1" 200 12492 "-" "Ansible" 1211 0.084 [auth-system-keycloak-80] [] 10.0.0.68:8080 12500 0.083 200 53f7221ffeb32bb41f298a3866092033 199.204.45.116 - - [29/Mar/2026:02:11:12 +0000] "GET /admin/realms/atmosphere/client-scopes/ddb8f85e-39e1-4a47-9f66-eb05d78d529a HTTP/1.1" 200 1278 "-" "Ansible" 1248 0.015 [auth-system-keycloak-80] [] 10.0.0.68:8080 1278 0.014 200 8c916a6f54bd4fc1de977bdfd241cfbb 199.204.45.116 - - [29/Mar/2026:02:11:12 +0000] "PUT /admin/realms/atmosphere/client-scopes/ddb8f85e-39e1-4a47-9f66-eb05d78d529a HTTP/1.1" 204 0 "-" "Ansible" 1848 0.042 [auth-system-keycloak-80] [] 10.0.0.68:8080 0 0.041 204 55778c3ad6f447ff63121daddc1fd6bf 199.204.45.116 - - [29/Mar/2026:02:11:13 +0000] "GET /admin/realms/atmosphere/client-scopes/ddb8f85e-39e1-4a47-9f66-eb05d78d529a/protocol-mappers/models HTTP/1.1" 200 963 "-" "Ansible" 1272 0.094 [auth-system-keycloak-80] [] 10.0.0.68:8080 963 0.093 200 2e60b61982bd8e404d8e65046b140ea0 199.204.45.116 - - [29/Mar/2026:02:11:13 +0000] "GET /admin/realms/atmosphere/client-scopes/ddb8f85e-39e1-4a47-9f66-eb05d78d529a/protocol-mappers/models/4a53b377-f79c-49a2-bf9b-1d7c0e4542b3 HTTP/1.1" 200 366 "-" "Ansible" 1309 0.008 [auth-system-keycloak-80] [] 10.0.0.68:8080 366 0.009 200 1bcc1d9910297f8bb59254ad143369cb 199.204.45.116 - - [29/Mar/2026:02:11:13 +0000] "PUT /admin/realms/atmosphere/client-scopes/ddb8f85e-39e1-4a47-9f66-eb05d78d529a/protocol-mappers/models/4a53b377-f79c-49a2-bf9b-1d7c0e4542b3 HTTP/1.1" 204 0 "-" "Ansible" 1622 0.072 [auth-system-keycloak-80] [] 10.0.0.68:8080 0 0.071 204 e25692d5704bef94590388454b605f2c 199.204.45.116 - - [29/Mar/2026:02:11:13 +0000] "GET /admin/realms/atmosphere/client-scopes/ddb8f85e-39e1-4a47-9f66-eb05d78d529a HTTP/1.1" 200 1218 "-" "Ansible" 1248 0.080 [auth-system-keycloak-80] [] 10.0.0.68:8080 1218 0.080 200 4f798585a7b47c9f7fc348e6458109ba W0329 02:11:19.316768 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:11:19.316803 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 199.204.45.116 - - [29/Mar/2026:02:11:34 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.475 [auth-system-keycloak-80] [] 10.0.0.68:8080 1928 0.474 200 1c26cec564b6cc00ea4d5ec6d0469757 199.204.45.116 - - [29/Mar/2026:02:11:34 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 2 "-" "Ansible" 1227 0.059 [auth-system-keycloak-80] [] 10.0.0.68:8080 2 0.059 200 f150ddc49357a72d4e3535807bb873a6 199.204.45.116 - - [29/Mar/2026:02:11:35 +0000] "POST /admin/realms/atmosphere/clients HTTP/1.1" 201 0 "-" "Ansible" 1657 0.279 [auth-system-keycloak-80] [] 10.0.0.68:8080 0 0.278 201 15b731d97df800469d36bb3ba65a9495 199.204.45.116 - - [29/Mar/2026:02:11:35 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 1261 "-" "Ansible" 1227 0.026 [auth-system-keycloak-80] [] 10.0.0.68:8080 1261 0.027 200 b051b99e1c797639ffc1a3861d724c95 199.204.45.116 - - [29/Mar/2026:02:11:35 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.441 [auth-system-keycloak-80] [] 10.0.0.68:8080 1928 0.440 200 12bf5293bc247aee572948b6d4fece3f 199.204.45.116 - - [29/Mar/2026:02:11:35 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 2 "-" "Ansible" 1222 0.023 [auth-system-keycloak-80] [] 10.0.0.68:8080 2 0.022 200 123a55ac00008b448d330558737743ef 199.204.45.116 - - [29/Mar/2026:02:11:36 +0000] "POST /admin/realms/atmosphere/clients HTTP/1.1" 201 0 "-" "Ansible" 1688 0.098 [auth-system-keycloak-80] [] 10.0.0.68:8080 0 0.097 201 49bcd33d02eeb49b2f2736141170b096 199.204.45.116 - - [29/Mar/2026:02:11:36 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.089 [auth-system-keycloak-80] [] 10.0.0.68:8080 1286 0.089 200 e731ba7bb5853e67e1b9129f37ae0d01 199.204.45.116 - - [29/Mar/2026:02:11:37 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.909 [auth-system-keycloak-80] [] 10.0.0.68:8080 1928 0.909 200 97ddf5eea21323017a8dddcfd8eed3be 199.204.45.116 - - [29/Mar/2026:02:11:37 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 2 "-" "Ansible" 1225 0.008 [auth-system-keycloak-80] [] 10.0.0.68:8080 2 0.009 200 9179569f2aa07bf9053771bc8b5bcd32 199.204.45.116 - - [29/Mar/2026:02:11:37 +0000] "POST /admin/realms/atmosphere/clients HTTP/1.1" 201 0 "-" "Ansible" 1649 0.540 [auth-system-keycloak-80] [] 10.0.0.68:8080 0 0.539 201 86c01a328a9d5665a86088b673b5aa5c 199.204.45.116 - - [29/Mar/2026:02:11:38 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 1251 "-" "Ansible" 1225 0.120 [auth-system-keycloak-80] [] 10.0.0.68:8080 1251 0.120 200 42db5743074e0ae8eedb8bb9be5c4d2b 199.204.45.116 - - [29/Mar/2026:02:11:39 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.483 [auth-system-keycloak-80] [] 10.0.0.68:8080 1928 0.482 200 ab632934075c7d46941306097c13cd54 199.204.45.116 - - [29/Mar/2026:02:11:39 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 1261 "-" "Ansible" 1227 0.007 [auth-system-keycloak-80] [] 10.0.0.68:8080 1261 0.007 200 4c6318bb98498d9bd3d3a83748c0996b 199.204.45.116 - - [29/Mar/2026:02:11:39 +0000] "GET /admin/realms/atmosphere/clients/06b3383c-6ddd-4f2d-a6da-13b66b9d2e44/roles/member HTTP/1.1" 404 119 "-" "Ansible" 1255 0.011 [auth-system-keycloak-80] [] 10.0.0.68:8080 119 0.011 404 5f47b2cb9a6e31fa196ab0c27ba668cb 199.204.45.116 - - [29/Mar/2026:02:11:39 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 1261 "-" "Ansible" 1227 0.005 [auth-system-keycloak-80] [] 10.0.0.68:8080 1261 0.004 200 81b06ae632e9bf888eed6c2e7c6177b5 199.204.45.116 - - [29/Mar/2026:02:11:39 +0000] "POST /admin/realms/atmosphere/clients/06b3383c-6ddd-4f2d-a6da-13b66b9d2e44/roles HTTP/1.1" 201 0 "-" "Ansible" 1350 0.039 [auth-system-keycloak-80] [] 10.0.0.68:8080 0 0.038 201 b79ddc003ad06773711a3a29bbb0b450 199.204.45.116 - - [29/Mar/2026:02:11:39 +0000] "GET /admin/realms/atmosphere/clients?clientId=alertmanager HTTP/1.1" 200 1261 "-" "Ansible" 1227 0.007 [auth-system-keycloak-80] [] 10.0.0.68:8080 1261 0.007 200 910be6298e6114ee95a0c25a72982779 199.204.45.116 - - [29/Mar/2026:02:11:39 +0000] "GET /admin/realms/atmosphere/clients/06b3383c-6ddd-4f2d-a6da-13b66b9d2e44/roles/member HTTP/1.1" 200 166 "-" "Ansible" 1255 0.016 [auth-system-keycloak-80] [] 10.0.0.68:8080 166 0.015 200 edf178aaa5201a1975e4d2f353aa2a96 199.204.45.116 - - [29/Mar/2026:02:11:40 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.479 [auth-system-keycloak-80] [] 10.0.0.68:8080 1928 0.477 200 a469a3e60530c18c3c2e8a1fd66aa67d 199.204.45.116 - - [29/Mar/2026:02:11:40 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.006 [auth-system-keycloak-80] [] 10.0.0.68:8080 1286 0.006 200 7c6296a3c6700226b71c34a74a9b6651 199.204.45.116 - - [29/Mar/2026:02:11:40 +0000] "GET /admin/realms/atmosphere/clients/ee340bc5-cfbe-4be1-ad76-f429c1fa754e/roles/admin HTTP/1.1" 404 119 "-" "Ansible" 1254 0.008 [auth-system-keycloak-80] [] 10.0.0.68:8080 119 0.007 404 c2c95bc0fea0f5e343a84d14c25ca60d 199.204.45.116 - - [29/Mar/2026:02:11:40 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.036 [auth-system-keycloak-80] [] 10.0.0.68:8080 1286 0.035 200 35daa481579d523a31a55e11186299e9 199.204.45.116 - - [29/Mar/2026:02:11:40 +0000] "POST /admin/realms/atmosphere/clients/ee340bc5-cfbe-4be1-ad76-f429c1fa754e/roles HTTP/1.1" 201 0 "-" "Ansible" 1349 0.013 [auth-system-keycloak-80] [] 10.0.0.68:8080 0 0.013 201 db3bf96fa889f68fb3623cea9f5c02f0 199.204.45.116 - - [29/Mar/2026:02:11:40 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.004 [auth-system-keycloak-80] [] 10.0.0.68:8080 1286 0.004 200 4d041ffc2f50fe20c33865be6a0a15be 199.204.45.116 - - [29/Mar/2026:02:11:40 +0000] "GET /admin/realms/atmosphere/clients/ee340bc5-cfbe-4be1-ad76-f429c1fa754e/roles/admin HTTP/1.1" 200 165 "-" "Ansible" 1254 0.009 [auth-system-keycloak-80] [] 10.0.0.68:8080 165 0.010 200 962b125bca4eac40fa90a66f8ee9bb40 199.204.45.116 - - [29/Mar/2026:02:11:40 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.468 [auth-system-keycloak-80] [] 10.0.0.68:8080 1928 0.467 200 4c357916b902aa9168479ef59c792f3a 199.204.45.116 - - [29/Mar/2026:02:11:41 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.007 [auth-system-keycloak-80] [] 10.0.0.68:8080 1286 0.008 200 62268b3d8641eeeafa510f9507617381 199.204.45.116 - - [29/Mar/2026:02:11:41 +0000] "GET /admin/realms/atmosphere/clients/ee340bc5-cfbe-4be1-ad76-f429c1fa754e/roles/editor HTTP/1.1" 404 119 "-" "Ansible" 1255 0.013 [auth-system-keycloak-80] [] 10.0.0.68:8080 119 0.014 404 b994a9300624e70a6097993be9f9dcb2 199.204.45.116 - - [29/Mar/2026:02:11:41 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.037 [auth-system-keycloak-80] [] 10.0.0.68:8080 1286 0.037 200 5443411b80a4a27ec5bc14f9eaca4bb8 199.204.45.116 - - [29/Mar/2026:02:11:41 +0000] "POST /admin/realms/atmosphere/clients/ee340bc5-cfbe-4be1-ad76-f429c1fa754e/roles HTTP/1.1" 201 0 "-" "Ansible" 1350 0.022 [auth-system-keycloak-80] [] 10.0.0.68:8080 0 0.021 201 c05a77a2c3beabafc9f9dec98d8f9873 199.204.45.116 - - [29/Mar/2026:02:11:41 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.057 [auth-system-keycloak-80] [] 10.0.0.68:8080 1286 0.056 200 e473d8e3244146ca151e7550dda039fb 199.204.45.116 - - [29/Mar/2026:02:11:41 +0000] "GET /admin/realms/atmosphere/clients/ee340bc5-cfbe-4be1-ad76-f429c1fa754e/roles/editor HTTP/1.1" 200 166 "-" "Ansible" 1255 0.014 [auth-system-keycloak-80] [] 10.0.0.68:8080 166 0.014 200 09d6ad61d82b5db88f9b0b3495d20295 199.204.45.116 - - [29/Mar/2026:02:11:42 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.526 [auth-system-keycloak-80] [] 10.0.0.68:8080 1928 0.525 200 e1f708abb528248e0f6380dbee67c997 199.204.45.116 - - [29/Mar/2026:02:11:42 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.047 [auth-system-keycloak-80] [] 10.0.0.68:8080 1286 0.047 200 b42be09e48b3819a3fb8da1869252e5e 199.204.45.116 - - [29/Mar/2026:02:11:42 +0000] "GET /admin/realms/atmosphere/clients/ee340bc5-cfbe-4be1-ad76-f429c1fa754e/roles/viewer HTTP/1.1" 404 119 "-" "Ansible" 1255 0.010 [auth-system-keycloak-80] [] 10.0.0.68:8080 119 0.010 404 308cfac4085510d7f891a8ddeeb9a209 199.204.45.116 - - [29/Mar/2026:02:11:42 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.008 [auth-system-keycloak-80] [] 10.0.0.68:8080 1286 0.008 200 658c57cebdb6fa8cc1957189069a4d70 199.204.45.116 - - [29/Mar/2026:02:11:42 +0000] "POST /admin/realms/atmosphere/clients/ee340bc5-cfbe-4be1-ad76-f429c1fa754e/roles HTTP/1.1" 201 0 "-" "Ansible" 1350 0.058 [auth-system-keycloak-80] [] 10.0.0.68:8080 0 0.058 201 7436c04782db662e4b65da8db599d173 199.204.45.116 - - [29/Mar/2026:02:11:42 +0000] "GET /admin/realms/atmosphere/clients?clientId=grafana HTTP/1.1" 200 1286 "-" "Ansible" 1222 0.007 [auth-system-keycloak-80] [] 10.0.0.68:8080 1286 0.008 200 d056a8820257f5404bce6dbe94e8dc43 199.204.45.116 - - [29/Mar/2026:02:11:42 +0000] "GET /admin/realms/atmosphere/clients/ee340bc5-cfbe-4be1-ad76-f429c1fa754e/roles/viewer HTTP/1.1" 200 166 "-" "Ansible" 1255 0.059 [auth-system-keycloak-80] [] 10.0.0.68:8080 166 0.059 200 22f7bb3e37fbcc9970db1e1507be2b19 199.204.45.116 - - [29/Mar/2026:02:11:43 +0000] "POST /realms/master/protocol/openid-connect/token HTTP/1.1" 200 1928 "-" "Ansible" 332 0.484 [auth-system-keycloak-80] [] 10.0.0.68:8080 1928 0.483 200 c508958d677de3a9eeeb167be80eecd2 199.204.45.116 - - [29/Mar/2026:02:11:43 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 1251 "-" "Ansible" 1225 0.027 [auth-system-keycloak-80] [] 10.0.0.68:8080 1251 0.027 200 74b3a245d371a7100af9d602f5f798e5 199.204.45.116 - - [29/Mar/2026:02:11:43 +0000] "GET /admin/realms/atmosphere/clients/e71dc783-5b7f-4b4a-a0d5-bf186f78ecfb/roles/member HTTP/1.1" 404 119 "-" "Ansible" 1255 0.009 [auth-system-keycloak-80] [] 10.0.0.68:8080 119 0.009 404 b8713159b818c5b013fa77db4d42499e 199.204.45.116 - - [29/Mar/2026:02:11:43 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 1251 "-" "Ansible" 1225 0.007 [auth-system-keycloak-80] [] 10.0.0.68:8080 1251 0.007 200 be71f47ff94817a1785f0f196af885fe 199.204.45.116 - - [29/Mar/2026:02:11:43 +0000] "POST /admin/realms/atmosphere/clients/e71dc783-5b7f-4b4a-a0d5-bf186f78ecfb/roles HTTP/1.1" 201 0 "-" "Ansible" 1350 0.057 [auth-system-keycloak-80] [] 10.0.0.68:8080 0 0.056 201 7340b0432b0ac05bb57c91038dfb24d1 199.204.45.116 - - [29/Mar/2026:02:11:43 +0000] "GET /admin/realms/atmosphere/clients?clientId=prometheus HTTP/1.1" 200 1251 "-" "Ansible" 1225 0.007 [auth-system-keycloak-80] [] 10.0.0.68:8080 1251 0.007 200 a89b04f84a2db8dc4743f704af27c139 199.204.45.116 - - [29/Mar/2026:02:11:43 +0000] "GET /admin/realms/atmosphere/clients/e71dc783-5b7f-4b4a-a0d5-bf186f78ecfb/roles/member HTTP/1.1" 200 166 "-" "Ansible" 1255 0.013 [auth-system-keycloak-80] [] 10.0.0.68:8080 166 0.012 200 98f95677fbd2e05281acb41fd7e97dbf W0329 02:12:59.413100 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:12:59.413150 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:12:59.794939 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:12:59.795010 7 controller.go:1441] Error getting SSL certificate "monitoring/prometheus-tls": local SSL certificate monitoring/prometheus-tls was not found. Using default certificate W0329 02:12:59.795043 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:12:59.795061 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:12:59.796299 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:12:59.796542 7 controller.go:1441] Error getting SSL certificate "monitoring/alertmanager-tls": local SSL certificate monitoring/alertmanager-tls was not found. Using default certificate W0329 02:12:59.796617 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:12:59.796713 7 controller.go:1441] Error getting SSL certificate "monitoring/grafana-tls": local SSL certificate monitoring/grafana-tls was not found. Using default certificate W0329 02:12:59.796765 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:12:59.796791 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0329 02:12:59.796809 7 admission.go:149] processed ingress via admission controller {testedIngressLength:2 testedIngressTime:0.002s renderingIngressLength:2 renderingIngressTime:0s admissionTime:0.002s testedConfigurationSize:18.2kB} I0329 02:12:59.796859 7 main.go:107] "successfully validated configuration, accepting" ingress="monitoring/kube-prometheus-stack-prometheus" W0329 02:12:59.796984 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:12:59.797014 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0329 02:12:59.802244 7 admission.go:149] processed ingress via admission controller {testedIngressLength:2 testedIngressTime:0.006s renderingIngressLength:2 renderingIngressTime:0.001s admissionTime:0.007s testedConfigurationSize:18.2kB} I0329 02:12:59.802307 7 main.go:107] "successfully validated configuration, accepting" ingress="monitoring/kube-prometheus-stack-alertmanager" I0329 02:12:59.802990 7 admission.go:149] processed ingress via admission controller {testedIngressLength:2 testedIngressTime:0.006s renderingIngressLength:2 renderingIngressTime:0.001s admissionTime:0.007s testedConfigurationSize:18.2kB} I0329 02:12:59.803036 7 main.go:107] "successfully validated configuration, accepting" ingress="monitoring/kube-prometheus-stack-grafana" I0329 02:12:59.808394 7 store.go:440] "Found valid IngressClass" ingress="monitoring/kube-prometheus-stack-prometheus" ingressclass="atmosphere" I0329 02:12:59.808770 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-prometheus", UID:"f85cc519-7f2f-42d4-8f2d-b4f809d0f8c3", APIVersion:"networking.k8s.io/v1", ResourceVersion:"4996", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0329 02:12:59.809019 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "monitoring/prometheus-tls" in local store I0329 02:12:59.812032 7 store.go:440] "Found valid IngressClass" ingress="monitoring/kube-prometheus-stack-alertmanager" ingressclass="atmosphere" W0329 02:12:59.812451 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "monitoring/alertmanager-tls" in local store I0329 02:12:59.812453 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-alertmanager", UID:"d96b47c3-7856-449d-91c7-609ee0f01213", APIVersion:"networking.k8s.io/v1", ResourceVersion:"4997", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync I0329 02:12:59.812474 7 store.go:440] "Found valid IngressClass" ingress="monitoring/kube-prometheus-stack-grafana" ingressclass="atmosphere" I0329 02:12:59.812638 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-grafana", UID:"bd8591c4-12ac-42de-a8bb-966e198f3980", APIVersion:"networking.k8s.io/v1", ResourceVersion:"4998", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0329 02:12:59.812827 7 backend_ssl.go:47] Error obtaining X.509 certificate: no object matching key "monitoring/grafana-tls" in local store I0329 02:13:00.324886 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="monitoring/grafana-tls" I0329 02:13:00.325884 7 backend_ssl.go:67] "Adding secret to local store" name="monitoring/grafana-tls" I0329 02:13:00.328058 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="monitoring/prometheus-tls" I0329 02:13:00.329541 7 backend_ssl.go:67] "Adding secret to local store" name="monitoring/prometheus-tls" I0329 02:13:01.476603 7 store.go:611] "Secret was added and it is used in ingress annotations. Parsing" secret="monitoring/alertmanager-tls" I0329 02:13:01.478553 7 backend_ssl.go:67] "Adding secret to local store" name="monitoring/alertmanager-tls" W0329 02:13:02.747219 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:02.747302 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:02.747342 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:02.747480 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:02.747513 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0329 02:13:02.747662 7 controller.go:196] "Configuration changes detected, backend reload required" I0329 02:13:02.808907 7 controller.go:216] "Backend successfully reloaded" I0329 02:13:02.809151 7 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-kbq2q", UID:"64a00c35-9bc1-4b5d-8b7a-ca800a9acc6a", APIVersion:"v1", ResourceVersion:"1552", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration W0329 02:13:06.080817 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:06.080900 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:06.080948 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:06.081093 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:06.081124 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store I0329 02:13:08.175154 7 status.go:304] "updating Ingress status" namespace="monitoring" ingress="kube-prometheus-stack-grafana" currentValue=null newValue=[{"ip":"10.97.77.34"}] I0329 02:13:08.175526 7 status.go:304] "updating Ingress status" namespace="monitoring" ingress="kube-prometheus-stack-alertmanager" currentValue=null newValue=[{"ip":"10.97.77.34"}] I0329 02:13:08.175726 7 status.go:304] "updating Ingress status" namespace="monitoring" ingress="kube-prometheus-stack-prometheus" currentValue=null newValue=[{"ip":"10.97.77.34"}] I0329 02:13:08.235025 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-grafana", UID:"bd8591c4-12ac-42de-a8bb-966e198f3980", APIVersion:"networking.k8s.io/v1", ResourceVersion:"5264", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync I0329 02:13:08.245861 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-alertmanager", UID:"d96b47c3-7856-449d-91c7-609ee0f01213", APIVersion:"networking.k8s.io/v1", ResourceVersion:"5265", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync I0329 02:13:08.248171 7 event.go:377] Event(v1.ObjectReference{Kind:"Ingress", Namespace:"monitoring", Name:"kube-prometheus-stack-prometheus", UID:"f85cc519-7f2f-42d4-8f2d-b4f809d0f8c3", APIVersion:"networking.k8s.io/v1", ResourceVersion:"5266", FieldPath:""}): type: 'Normal' reason: 'Sync' Scheduled for sync W0329 02:13:09.414131 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:09.414215 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:09.414255 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:09.414379 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:09.414413 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:12.747657 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:12.747730 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:12.747776 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:12.747911 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:12.747939 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:16.080016 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:16.080066 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:16.080103 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:16.080199 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:16.080221 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:19.414098 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:19.414200 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:19.414252 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:19.414436 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:19.414488 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:22.748566 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:22.748680 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:22.748756 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:22.748938 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:22.748980 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:26.080084 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:26.080127 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:26.080150 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:26.080214 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:26.080230 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:29.414308 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:29.414372 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:29.414410 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:29.414528 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:29.414561 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:32.748381 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:32.748440 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:32.748467 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:32.748559 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:32.748578 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:36.081362 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:36.081424 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:36.081448 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:36.081538 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:36.081560 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:39.414039 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:39.414120 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:39.414163 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:39.414295 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:39.414330 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:42.746763 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:42.746845 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:42.746885 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:42.747014 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:42.747045 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:46.080650 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:46.080721 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:46.080761 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:46.080915 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:46.080946 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:49.977816 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:49.977884 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:49.977925 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:49.978067 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:49.978097 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:53.312811 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:13:53.312964 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:13:53.313007 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:13:53.313179 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:13:53.313207 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:10.005735 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:14:10.005850 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:14:10.005901 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:14:10.007284 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:10.007383 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:13.339061 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:14:13.339146 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:14:13.339200 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:14:13.339341 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:13.339371 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:30.547284 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:14:30.547360 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:14:30.547432 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:14:30.547546 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:30.547567 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:33.881364 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:14:33.881427 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:14:33.881474 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:14:33.881600 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:33.881633 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:37.214862 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:14:37.214930 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-grafana" does not have any active Endpoint. W0329 02:14:37.214970 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:14:37.215190 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:37.215270 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:40.547504 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:14:40.547611 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:14:40.547754 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:40.547786 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:43.881167 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:14:43.881286 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:14:43.881463 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:43.881496 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:47.214191 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:14:47.214311 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:14:47.214466 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:47.214499 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:50.547829 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:14:50.547947 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:14:50.548472 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:50.548511 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:57.388678 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:14:57.388783 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:14:57.388890 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:14:57.388912 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store 10.0.0.230 - - [29/Mar/2026:02:14:57 +0000] "GET /realms/atmosphere/.well-known/openid-configuration HTTP/1.1" 200 6432 "-" "Go-http-client/1.1" 161 0.032 [auth-system-keycloak-80] [] 10.0.0.68:8080 6432 0.033 200 1aa415a8b8c4418103a66535eeaf366b W0329 02:15:00.722445 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-prometheus" does not have any active Endpoint. W0329 02:15:00.722550 7 controller.go:1219] Service "monitoring/kube-prometheus-stack-alertmanager" does not have any active Endpoint. W0329 02:15:00.722678 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:15:00.722739 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store E0329 02:15:07.157256 7 leaderelection.go:429] Failed to update lock optimistically: etcdserver: request timed out, falling back to slow path E0329 02:15:15.152380 7 leaderelection.go:436] error retrieving resource lock ingress-nginx/ingress-nginx-leader: Get "https://10.96.0.1:443/apis/coordination.k8s.io/v1/namespaces/ingress-nginx/leases/ingress-nginx-leader": context deadline exceeded I0329 02:15:15.152432 7 leaderelection.go:297] failed to renew lease ingress-nginx/ingress-nginx-leader: context deadline exceeded E0329 02:15:15.152637 7 status.go:104] "error running poll" err="timed out waiting for the condition" I0329 02:15:15.152638 7 leaderelection.go:257] attempting to acquire leader lease ingress-nginx/ingress-nginx-leader... E0329 02:15:18.879832 7 leaderelection.go:429] Failed to update lock optimistically: Operation cannot be fulfilled on leases.coordination.k8s.io "ingress-nginx-leader": the object has been modified; please apply your changes to the latest version and try again, falling back to slow path I0329 02:15:20.880647 7 leaderelection.go:271] successfully acquired lease ingress-nginx/ingress-nginx-leader 10.0.0.123 - - [29/Mar/2026:02:15:34 +0000] "GET /realms/atmosphere/.well-known/openid-configuration HTTP/1.1" 200 6432 "-" "Go-http-client/1.1" 161 0.008 [auth-system-keycloak-80] [] 10.0.0.68:8080 6432 0.007 200 791c45cd3e0f21fd5effdb54f228027d W0329 02:16:01.431168 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:16:01.431227 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:16:04.764691 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:16:04.764739 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:16:08.098005 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:16:08.098066 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:16:30.792757 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store W0329 02:16:30.792818 7 controller.go:512] Error getting Service "openstack/minidns": no object matching key "openstack/minidns" in local store