Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2341/e2e-tests/logs/upgrade-consistency-sharded-tls.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- grep: warning: stray \ before - grep: warning: stray \ before - error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- error: resource(s) were provided, but no name was specified ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- namespace/psmdb-operator created Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2341-07f38ccbd-10-cluster14" modified. ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2341-07f38ccbd ----------------------------------------------------------------------------------- customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created deployment.apps/percona-server-mongodb-operator created waiting for pod/percona-server-mongodb-operator-fd5cdb985-z7tkk to be ready.OK Print operator info from log 2026-06-11T08:09:16.013Z INFO setup Manager starting up {"gitCommit": "07f38ccbd4f31eaf304829687e6a4b654579c3aa", "gitBranch": "PR-2341-07f38ccbd", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified error: resource(s) were provided, but no name was specified ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- cleaned up old namespaces upgrade-consistency-sharded-tls-24031 ----------------------------------------------------------------------------------- error: resource(s) were provided, but no name was specified ----------------------------------------------------------------------------------- create namespace upgrade-consistency-sharded-tls-24031 ----------------------------------------------------------------------------------- namespace/upgrade-consistency-sharded-tls-24031 created Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2341-07f38ccbd-10-cluster14" modified. ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- namespace/cert-manager created namespace/cert-manager labeled namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io created serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view created clusterrole.rbac.authorization.k8s.io/cert-manager-view created clusterrole.rbac.authorization.k8s.io/cert-manager-edit created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created role.rbac.authorization.k8s.io/cert-manager:leaderelection created role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. pod/cert-manager-559d798845-9qjtb condition met pod/cert-manager-cainjector-64958d9c7c-kpj9c condition met pod/cert-manager-webhook-7fb6f99b56-ckgth condition met E0611 08:10:11.591256 252832 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/api/v1/namespaces/cert-manager/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcert-manager-cainjector-64958d9c7c-kpj9c&resourceVersion=1781165411218271016&timeoutSeconds=375&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- secret/some-users created deployment.apps/psmdb-client created role.rbac.authorization.k8s.io/cmctl created serviceaccount/cmctl created rolebinding.rbac.authorization.k8s.io/service-account-cmctl created deployment.apps/cmctl created ----------------------------------------------------------------------------------- create first PSMDB cluster 1.21.2 some-name ----------------------------------------------------------------------------------- perconaservermongodb.psmdb.percona.com/some-name created ----------------------------------------------------------------------------------- check if Pod started ----------------------------------------------------------------------------------- waiting for pod/some-name-rs0-0 to be ready....................OK waiting for pod/some-name-rs0-1 to be ready.............OK waiting for pod/some-name-rs0-2 to be ready..................OK Waiting for cluster readyness........................ waiting for pod/some-name-cfg-0 to be ready.OK waiting for pod/some-name-cfg-1 to be ready.OK waiting for pod/some-name-cfg-2 to be ready.OK Waiting for cluster readyness waiting for pod/some-name-mongos-0 to be ready.OK waiting for pod/some-name-mongos-1 to be ready.OK waiting for pod/some-name-mongos-2 to be ready.OK Waiting for cluster readyness waiting for cluster readyness.OK ----------------------------------------------------------------------------------- check if Pod started ----------------------------------------------------------------------------------- waiting for pod/some-name-rs0-0 to be ready.OK waiting for pod/some-name-rs0-1 to be ready.OK waiting for pod/some-name-rs0-2 to be ready.OK Waiting for cluster readyness waiting for pod/some-name-cfg-0 to be ready.OK waiting for pod/some-name-cfg-1 to be ready.OK waiting for pod/some-name-cfg-2 to be ready.OK Waiting for cluster readyness waiting for pod/some-name-mongos-0 to be ready.OK waiting for pod/some-name-mongos-1 to be ready.OK waiting for pod/some-name-mongos-2 to be ready.OK Waiting for cluster readyness waiting for cluster readyness.OK E0611 08:17:30.939185 327072 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781165846679800000&timeoutSeconds=493&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:17:33.224061 327305 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781165851679870000&timeoutSeconds=541&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:17:35.571093 327527 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781165851679870000&timeoutSeconds=539&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:17:37.928911 327924 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781165856679890000&timeoutSeconds=344&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:17:40.331836 328163 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781165856679890000&timeoutSeconds=344&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:17:42.686379 328398 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781165861679930000&timeoutSeconds=449&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:17:44.965331 328578 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781165861679930000&timeoutSeconds=444&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:17:47.283591 328804 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781165866679954000&timeoutSeconds=379&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:17:49.600883 329135 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781165866679954000&timeoutSeconds=329&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:17:51.908548 329328 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781165871679965000&timeoutSeconds=394&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met ----------------------------------------------------------------------------------- renew some-name-ssl ----------------------------------------------------------------------------------- Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-24031/some-name-ssl waiting for pod/some-name-rs0-0 to be ready.OK waiting for pod/some-name-rs0-1 to be ready.OK waiting for pod/some-name-rs0-2 to be ready.OK Waiting for cluster readyness...................................................................................................................................... waiting for pod/some-name-cfg-0 to be ready.OK waiting for pod/some-name-cfg-1 to be ready.OK waiting for pod/some-name-cfg-2 to be ready.OK Waiting for cluster readyness waiting for pod/some-name-mongos-0 to be ready.OK waiting for pod/some-name-mongos-1 to be ready.OK waiting for pod/some-name-mongos-2 to be ready.OK Waiting for cluster readyness waiting for cluster readyness.OK E0611 08:24:02.352712 368055 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781166241698498000&timeoutSeconds=306&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:24:04.674157 368405 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781166241698498000&timeoutSeconds=514&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:24:07.014810 368771 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781166246698515000&timeoutSeconds=434&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:24:09.279702 369004 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781166246698515000&timeoutSeconds=372&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:24:11.523498 369292 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781166246698515000&timeoutSeconds=402&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:24:13.938075 369574 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781166251698528000&timeoutSeconds=309&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:24:16.243804 369935 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781166251698528000&timeoutSeconds=375&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:24:18.557821 370211 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781166256698728000&timeoutSeconds=473&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:24:20.896901 370479 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781166256698728000&timeoutSeconds=427&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:24:23.391881 370720 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781166261698735000&timeoutSeconds=579&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met ----------------------------------------------------------------------------------- renew some-name-ssl-internal ----------------------------------------------------------------------------------- Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-24031/some-name-ssl-internal waiting for pod/some-name-rs0-0 to be ready.OK waiting for pod/some-name-rs0-1 to be ready.OK waiting for pod/some-name-rs0-2 to be ready.OK Waiting for cluster readyness......................................................................................................................................... waiting for pod/some-name-cfg-0 to be ready.OK waiting for pod/some-name-cfg-1 to be ready.OK waiting for pod/some-name-cfg-2 to be ready.OK Waiting for cluster readyness waiting for pod/some-name-mongos-0 to be ready.OK waiting for pod/some-name-mongos-1 to be ready.OK waiting for pod/some-name-mongos-2 to be ready.OK Waiting for cluster readyness waiting for cluster readyness.OK ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- [2026-06-11T08:30:30+0000] compare_kubectl: service/some-name-rs0 OK [2026-06-11T08:30:31+0000] compare_kubectl: service/some-name-cfg OK [2026-06-11T08:30:32+0000] compare_kubectl: statefulset/some-name-rs0 OK [2026-06-11T08:30:33+0000] compare_kubectl: statefulset/some-name-cfg OK ----------------------------------------------------------------------------------- test 1.22.0 ----------------------------------------------------------------------------------- perconaservermongodb.psmdb.percona.com/some-name patched ----------------------------------------------------------------------------------- check if Pod started ----------------------------------------------------------------------------------- waiting for pod/some-name-rs0-0 to be ready.OK waiting for pod/some-name-rs0-1 to be ready.OK waiting for pod/some-name-rs0-2 to be ready.OK Waiting for cluster readyness............................................................................................. waiting for pod/some-name-cfg-0 to be ready.OK waiting for pod/some-name-cfg-1 to be ready.OK waiting for pod/some-name-cfg-2 to be ready.OK Waiting for cluster readyness waiting for pod/some-name-mongos-0 to be ready.OK waiting for pod/some-name-mongos-1 to be ready.OK waiting for pod/some-name-mongos-2 to be ready.OK Waiting for cluster readyness waiting for cluster readyness.OK E0611 08:35:07.470014 422550 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781166902900125000&timeoutSeconds=462&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:35:09.827420 422611 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781166907900227000&timeoutSeconds=538&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:35:12.159224 422697 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781166907900227000&timeoutSeconds=589&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:35:13.916816 422827 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781166912900279000&timeoutSeconds=350&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:35:16.238665 422895 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781166912900279000&timeoutSeconds=519&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:35:18.596919 423017 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781166917900661000&timeoutSeconds=375&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:35:20.974585 423144 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781166917900661000&timeoutSeconds=515&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:35:23.355823 423273 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781166922900712000&timeoutSeconds=462&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:35:25.670651 423406 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781166922900712000&timeoutSeconds=329&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:35:27.980150 423569 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781166922900712000&timeoutSeconds=479&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met ----------------------------------------------------------------------------------- renew some-name-ssl ----------------------------------------------------------------------------------- Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-24031/some-name-ssl waiting for pod/some-name-rs0-0 to be ready.OK waiting for pod/some-name-rs0-1 to be ready.OK waiting for pod/some-name-rs0-2 to be ready.OK Waiting for cluster readyness........................................................................................................................................... waiting for pod/some-name-cfg-0 to be ready.OK waiting for pod/some-name-cfg-1 to be ready.OK waiting for pod/some-name-cfg-2 to be ready.OK Waiting for cluster readyness waiting for pod/some-name-mongos-0 to be ready.OK waiting for pod/some-name-mongos-1 to be ready.OK waiting for pod/some-name-mongos-2 to be ready.OK Waiting for cluster readyness waiting for cluster readyness.OK E0611 08:41:34.945370 441771 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781167292646174000&timeoutSeconds=526&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:41:37.318505 441907 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781167292646174000&timeoutSeconds=578&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:41:39.620607 442032 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781167297646349000&timeoutSeconds=300&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:41:41.965736 442114 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781167297646349000&timeoutSeconds=540&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:41:44.286418 442240 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781167302646358000&timeoutSeconds=576&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:41:46.602839 442337 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781167302646358000&timeoutSeconds=507&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:41:48.976476 442439 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781167307646727000&timeoutSeconds=365&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:41:51.353132 442566 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781167307646727000&timeoutSeconds=438&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:41:53.708870 442672 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781167312646733000&timeoutSeconds=349&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:41:55.982347 442859 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781167312646733000&timeoutSeconds=522&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met ----------------------------------------------------------------------------------- renew some-name-ssl-internal ----------------------------------------------------------------------------------- Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-24031/some-name-ssl-internal waiting for pod/some-name-rs0-0 to be ready.OK waiting for pod/some-name-rs0-1 to be ready.OK waiting for pod/some-name-rs0-2 to be ready.OK Waiting for cluster readyness............................................................................................................................................................. waiting for pod/some-name-cfg-0 to be ready.OK waiting for pod/some-name-cfg-1 to be ready.OK waiting for pod/some-name-cfg-2 to be ready.OK Waiting for cluster readyness waiting for pod/some-name-mongos-0 to be ready.OK waiting for pod/some-name-mongos-1 to be ready.OK waiting for pod/some-name-mongos-2 to be ready.OK Waiting for cluster readyness waiting for cluster readyness.OK ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- [2026-06-11T08:47:47+0000] compare_kubectl: service/some-name-rs0 OK [2026-06-11T08:47:48+0000] compare_kubectl: service/some-name-cfg OK [2026-06-11T08:47:49+0000] compare_kubectl: statefulset/some-name-rs0 OK [2026-06-11T08:47:49+0000] compare_kubectl: statefulset/some-name-cfg OK ----------------------------------------------------------------------------------- test 1.23.0 ----------------------------------------------------------------------------------- perconaservermongodb.psmdb.percona.com/some-name patched ----------------------------------------------------------------------------------- check if Pod started ----------------------------------------------------------------------------------- waiting for pod/some-name-rs0-0 to be ready.OK waiting for pod/some-name-rs0-1 to be ready.OK waiting for pod/some-name-rs0-2 to be ready.OK Waiting for cluster readyness waiting for pod/some-name-cfg-0 to be ready.OK waiting for pod/some-name-cfg-1 to be ready.OK waiting for pod/some-name-cfg-2 to be ready.OK Waiting for cluster readyness waiting for pod/some-name-mongos-0 to be ready.OK waiting for pod/some-name-mongos-1 to be ready.OK waiting for pod/some-name-mongos-2 to be ready.OK Waiting for cluster readyness waiting for cluster readyness.OK E0611 08:49:02.082722 454394 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781167740636235000&timeoutSeconds=419&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:49:03.849644 454414 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781167740636235000&timeoutSeconds=530&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:49:05.616072 454434 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781167740636235000&timeoutSeconds=545&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:49:07.377398 454455 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781167745636250000&timeoutSeconds=498&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:49:09.108506 454475 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781167745636250000&timeoutSeconds=325&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:49:10.872524 454497 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781167750636257000&timeoutSeconds=479&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:49:12.625213 454519 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781167750636257000&timeoutSeconds=466&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:49:14.382841 454540 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781167750636257000&timeoutSeconds=552&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:49:16.143288 454560 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781167755636305000&timeoutSeconds=491&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met E0611 08:49:17.903664 454581 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781167755636305000&timeoutSeconds=470&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met ----------------------------------------------------------------------------------- renew some-name-ssl ----------------------------------------------------------------------------------- Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-24031/some-name-ssl waiting for pod/some-name-rs0-0 to be ready.OK waiting for pod/some-name-rs0-1 to be ready.OK waiting for pod/some-name-rs0-2 to be ready.OK Waiting for cluster readyness.................................................................................................................................................................... waiting for pod/some-name-cfg-0 to be ready.OK waiting for pod/some-name-cfg-1 to be ready.OK waiting for pod/some-name-cfg-2 to be ready.OK Waiting for cluster readyness waiting for pod/some-name-mongos-0 to be ready.OK waiting for pod/some-name-mongos-1 to be ready.OK waiting for pod/some-name-mongos-2 to be ready.OK Waiting for cluster readyness waiting for cluster readyness.OK E0611 08:54:52.207895 459534 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781168091145110000&timeoutSeconds=406&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:54:53.966510 459552 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781168091145110000&timeoutSeconds=397&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:54:55.701303 459575 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781168091145110000&timeoutSeconds=317&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:54:57.456062 459595 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781168096145219000&timeoutSeconds=383&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:54:59.218275 459617 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781168096145219000&timeoutSeconds=304&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:55:00.979815 459637 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781168096145219000&timeoutSeconds=427&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:55:02.736735 459655 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781168101145225000&timeoutSeconds=314&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:55:04.467459 459676 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781168101145225000&timeoutSeconds=447&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:55:06.228223 459697 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781168101145225000&timeoutSeconds=515&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met E0611 08:55:07.989690 459718 reflector.go:227] "Failed to watch" err="Get \"https://34.57.128.4/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-24031/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781168106145241000&timeoutSeconds=312&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met ----------------------------------------------------------------------------------- renew some-name-ssl-internal ----------------------------------------------------------------------------------- Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-24031/some-name-ssl-internal waiting for pod/some-name-rs0-0 to be ready.OK waiting for pod/some-name-rs0-1 to be ready.OK waiting for pod/some-name-rs0-2 to be ready.OK Waiting for cluster readyness........................................................................................................................................................................ waiting for pod/some-name-cfg-0 to be ready.OK waiting for pod/some-name-cfg-1 to be ready.OK waiting for pod/some-name-cfg-2 to be ready.OK Waiting for cluster readyness waiting for pod/some-name-mongos-0 to be ready.OK waiting for pod/some-name-mongos-1 to be ready.OK waiting for pod/some-name-mongos-2 to be ready.OK Waiting for cluster readyness waiting for cluster readyness.OK ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- [2026-06-11T09:00:49+0000] compare_kubectl: service/some-name-rs0 OK [2026-06-11T09:00:49+0000] compare_kubectl: service/some-name-cfg OK [2026-06-11T09:00:50+0000] compare_kubectl: statefulset/some-name-rs0 OK [2026-06-11T09:00:51+0000] compare_kubectl: statefulset/some-name-cfg OK ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- ----------------------------------------------------------------------------------- Delete psmdb-backup ----------------------------------------------------------------------------------- No resources found in upgrade-consistency-sharded-tls-24031 namespace. ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted grep: warning: stray \ before - grep: warning: stray \ before - error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbs.psmdb.percona.com -n upgrade-consistency-sharded-tls-24031 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodb.psmdb.percona.com/some-name patched customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted namespace "cert-manager" deleted customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted serviceaccount "cert-manager-cainjector" deleted from cert-manager namespace serviceaccount "cert-manager" deleted from cert-manager namespace serviceaccount "cert-manager-webhook" deleted from cert-manager namespace clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace role.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace rolebinding.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace service "cert-manager-cainjector" deleted from cert-manager namespace service "cert-manager" deleted from cert-manager namespace service "cert-manager-webhook" deleted from cert-manager namespace deployment.apps "cert-manager-cainjector" deleted from cert-manager namespace deployment.apps "cert-manager" deleted from cert-manager namespace deployment.apps "cert-manager-webhook" deleted from cert-manager namespace mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------