Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/logs/monitoring-2-0-8-0.log
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ cluster=monitoring
+ create_infra monitoring-2-0-932
+ local ns=monitoring-2-0-932
+ '[' -n pxc-operator ']'
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl get pxc --all-namespaces -o wide
+ kubectl patch pxc -n monitoring-2-0-29557 monitoring --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/monitoring patched (no change)
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.gfhR11nJ0w
++ mktemp
+ local LAST_ERR=/tmp/tmp.7uL8QRDHRs
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gfhR11nJ0w
perconaxtradbcluster.pxc.percona.com "monitoring" deleted from monitoring-2-0-29557 namespace
+ cat /tmp/tmp.7uL8QRDHRs
+ rm /tmp/tmp.gfhR11nJ0w /tmp/tmp.7uL8QRDHRs
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.HUlixe52sk
++ mktemp
+ local LAST_ERR=/tmp/tmp.RU488jrDJ2
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HUlixe52sk
No resources found
+ cat /tmp/tmp.RU488jrDJ2
+ rm /tmp/tmp.HUlixe52sk /tmp/tmp.RU488jrDJ2
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.X3DyCkyrwN
++ mktemp
+ local LAST_ERR=/tmp/tmp.0X1B2BqvkG
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.X3DyCkyrwN
No resources found
+ cat /tmp/tmp.0X1B2BqvkG
+ rm /tmp/tmp.X3DyCkyrwN /tmp/tmp.0X1B2BqvkG
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ awk '-F ' '{print $2}'
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.YFi9zVJFk6
+ kubectl_bin get ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.8ejt89syNn
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ awk '{print$1}'
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.WcByjDo49q
+ xargs kubectl delete ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.uV0Jdoo4rJ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WcByjDo49q
+ cat /tmp/tmp.uV0Jdoo4rJ
+ rm /tmp/tmp.WcByjDo49q /tmp/tmp.uV0Jdoo4rJ
+ return 0
namespace "monitoring-2-0-29557" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.YFi9zVJFk6
namespace "pxc-operator" deleted
+ cat /tmp/tmp.8ejt89syNn
+ rm /tmp/tmp.YFi9zVJFk6 /tmp/tmp.8ejt89syNn
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.StiobhE06E
++ mktemp
+ local LAST_ERR=/tmp/tmp.iQmrzV5T6I
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.StiobhE06E
namespace/pxc-operator created
+ cat /tmp/tmp.iQmrzV5T6I
+ rm /tmp/tmp.StiobhE06E /tmp/tmp.iQmrzV5T6I
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rqCh1bLwHp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tZY2S9A2dm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rqCh1bLwHp
++ cat /tmp/tmp.tZY2S9A2dm
++ rm /tmp/tmp.rqCh1bLwHp /tmp/tmp.tZY2S9A2dm
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2213-6c08ea71-7-cluster9 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.45nnYv8awq
++ mktemp
+ local LAST_ERR=/tmp/tmp.RRtXjoqalr
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2213-6c08ea71-7-cluster9 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.45nnYv8awq
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2213-6c08ea71-7-cluster9" modified.
+ cat /tmp/tmp.RRtXjoqalr
+ rm /tmp/tmp.45nnYv8awq /tmp/tmp.RRtXjoqalr
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.YuBYyMh6HP
++ mktemp
+ local LAST_ERR=/tmp/tmp.LPyZ8fosAv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.YuBYyMh6HP
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.LPyZ8fosAv
+ rm /tmp/tmp.YuBYyMh6HP /tmp/tmp.LPyZ8fosAv
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/deploy/cw-rbac.yaml
+ kubectl_bin apply -f -
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
++ mktemp
+ local LAST_OUT=/tmp/tmp.i2nnXxGcV5
++ mktemp
+ local LAST_ERR=/tmp/tmp.AwwD1lbPch
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.i2nnXxGcV5
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.AwwD1lbPch
+ rm /tmp/tmp.i2nnXxGcV5 /tmp/tmp.AwwD1lbPch
+ return 0
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/deploy/cw-operator.yaml
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ kubectl_bin apply -f -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2213-6c08ea71^'
++ mktemp
+ local LAST_OUT=/tmp/tmp.iYT0HCKjQK
++ mktemp
+ local LAST_ERR=/tmp/tmp.c1TjcLBXPO
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.iYT0HCKjQK
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.c1TjcLBXPO
+ rm /tmp/tmp.iYT0HCKjQK /tmp/tmp.c1TjcLBXPO
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.OujsYxfdxC
++ mktemp
+ local LAST_ERR=/tmp/tmp.Pnn96estuP
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.OujsYxfdxC
pod/percona-xtradb-cluster-operator-f5b849cf6-jlmmq condition met
+ cat /tmp/tmp.Pnn96estuP
+ rm /tmp/tmp.OujsYxfdxC /tmp/tmp.Pnn96estuP
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SSMD7UvXyw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.282TRmdmo8
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SSMD7UvXyw
++ cat /tmp/tmp.282TRmdmo8
++ rm /tmp/tmp.SSMD7UvXyw /tmp/tmp.282TRmdmo8
++ return 0
+ wait_pod percona-xtradb-cluster-operator-f5b849cf6-jlmmq 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-f5b849cf6-jlmmq
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-f5b849cf6-jlmmq
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-f5b849cf6-jlmmq condition met
waiting for pod/percona-xtradb-cluster-operator-f5b849cf6-jlmmq to become Ready.Ok
+ sleep 3
+ create_namespace monitoring-2-0-932
+ local namespace=monitoring-2-0-932
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl api-resources
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrolebinding
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrole
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces monitoring-2-0-932'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces monitoring-2-0-932
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace monitoring-2-0-932
+ xargs kubectl delete ns
+ awk '{print$1}'
+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.0VB245YkxI
++ mktemp
+ local LAST_OUT=/tmp/tmp.i0ydurxMJd
+ local LAST_ERR=/tmp/tmp.CBrmkXBOIJ
+ local exit_status=0
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.Nsl04imgIx
+ local exit_status=0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace monitoring-2-0-932
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace monitoring-2-0-932
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0VB245YkxI
+ cat /tmp/tmp.CBrmkXBOIJ
+ rm /tmp/tmp.0VB245YkxI /tmp/tmp.CBrmkXBOIJ
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace monitoring-2-0-932
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.i0ydurxMJd
+ cat /tmp/tmp.Nsl04imgIx
Error from server (NotFound): namespaces "monitoring-2-0-932" not found
+ rm /tmp/tmp.i0ydurxMJd /tmp/tmp.Nsl04imgIx
+ return 1
+ :
+ wait_for_delete namespace/monitoring-2-0-932
+ local res=namespace/monitoring-2-0-932
+ echo -n 'waiting for namespace/monitoring-2-0-932 to be deleted'
waiting for namespace/monitoring-2-0-932 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "monitoring-2-0-932" not found
+ desc 'create namespace monitoring-2-0-932'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace monitoring-2-0-932
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace monitoring-2-0-932
++ mktemp
+ local LAST_OUT=/tmp/tmp.iYUPZv1iZC
++ mktemp
+ local LAST_ERR=/tmp/tmp.a5beRLwGE4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace monitoring-2-0-932
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.iYUPZv1iZC
namespace/monitoring-2-0-932 created
+ cat /tmp/tmp.a5beRLwGE4
+ rm /tmp/tmp.iYUPZv1iZC /tmp/tmp.a5beRLwGE4
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.s1X1p9Uthz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OU7s9uuBSf
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.s1X1p9Uthz
++ cat /tmp/tmp.OU7s9uuBSf
++ rm /tmp/tmp.s1X1p9Uthz /tmp/tmp.OU7s9uuBSf
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2213-6c08ea71-7-cluster9 --namespace=monitoring-2-0-932
++ mktemp
+ local LAST_OUT=/tmp/tmp.AkfJ7Aoj1Z
++ mktemp
+ local LAST_ERR=/tmp/tmp.0bDgp7K8tG
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2213-6c08ea71-7-cluster9 --namespace=monitoring-2-0-932
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.AkfJ7Aoj1Z
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2213-6c08ea71-7-cluster9" modified.
+ cat /tmp/tmp.0bDgp7K8tG
+ rm /tmp/tmp.AkfJ7Aoj1Z /tmp/tmp.0bDgp7K8tG
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.2dbQQQycDA
++ mktemp
+ local LAST_ERR=/tmp/tmp.a1q8vGKh3V
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2dbQQQycDA
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.a1q8vGKh3V
+ rm /tmp/tmp.2dbQQQycDA /tmp/tmp.a1q8vGKh3V
+ return 0
+ deploy_helm monitoring-2-0-932
+ helm repo add hashicorp https://helm.releases.hashicorp.com
"hashicorp" already exists with the same configuration, skipping
+ helm repo add minio https://charts.min.io/
"minio" already exists with the same configuration, skipping
+ helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "chaos-mesh" chart repository
...Successfully got an update from the "minio" chart repository
...Successfully got an update from the "percona" chart repository
...Successfully got an update from the "hashicorp" chart repository
Update Complete. ⎈Happy Helming!⎈
+ desc 'install PMM Server'
+ set +o xtrace


-----------------------------------------------------------------------------------
install PMM Server
-----------------------------------------------------------------------------------

+ deploy_pmm_server
+ '[' '!' -z '' ']'
+ helm repo add percona https://percona.github.io/percona-helm-charts/
"percona" already exists with the same configuration, skipping
+ helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "chaos-mesh" chart repository
...Successfully got an update from the "minio" chart repository
...Successfully got an update from the "percona" chart repository
...Successfully got an update from the "hashicorp" chart repository
Update Complete. ⎈Happy Helming!⎈
+ helm uninstall monitoring
Error: uninstall: Release not loaded: monitoring: release: not found
+ :
+ retry 10 60 helm install monitoring --set imageRepo=perconalab/pmm-server --set imageTag=dev-latest https://percona-charts.storage.googleapis.com/pmm-server-9.9.9.tgz
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install monitoring --set imageRepo=perconalab/pmm-server --set imageTag=dev-latest https://percona-charts.storage.googleapis.com/pmm-server-9.9.9.tgz
NAME: monitoring
LAST DEPLOYED: Wed Oct 15 10:25:10 2025
NAMESPACE: monitoring-2-0-932
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
PMM server can be accessed via HTTPS (port 443) on the following DNS name from within your cluster:

endpoint: https://monitoring-service.monitoring-2-0-932.svc.cluster.local:443
login:    admin
password: admin
+ kubectl wait pod monitoring-0 --for=condition=Ready --timeout=420s
pod/monitoring-0 condition met
+ kubectl_bin wait --for=condition=Ready pod/monitoring-0 --timeout=120s
++ mktemp
+ local LAST_OUT=/tmp/tmp.b8eblxwXiU
++ mktemp
+ local LAST_ERR=/tmp/tmp.jpIv7128Pi
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod/monitoring-0 --timeout=120s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.b8eblxwXiU
pod/monitoring-0 condition met
+ cat /tmp/tmp.jpIv7128Pi
+ rm /tmp/tmp.b8eblxwXiU /tmp/tmp.jpIv7128Pi
+ return 0
+ kubectl_bin exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
++ mktemp
+ local LAST_OUT=/tmp/tmp.8hetrN8nRn
++ mktemp
+ local LAST_ERR=/tmp/tmp.p9a0bWXSf3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8hetrN8nRn
+ cat /tmp/tmp.p9a0bWXSf3
+ rm /tmp/tmp.8hetrN8nRn /tmp/tmp.p9a0bWXSf3
+ return 0
++ kubectl_bin exec monitoring-0 -- bash -c 'printenv | grep ADMIN_PASSWORD | cut -d '\''='\'' -f2'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NBrwS7y7nW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ajXL9jfeYg
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec monitoring-0 -- bash -c 'printenv | grep ADMIN_PASSWORD | cut -d '\''='\'' -f2'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NBrwS7y7nW
++ cat /tmp/tmp.ajXL9jfeYg
++ rm /tmp/tmp.NBrwS7y7nW /tmp/tmp.ajXL9jfeYg
++ return 0
+ ADMIN_PASSWORD=admin
+ sleep 5
+ kubectl_bin exec monitoring-0 -- bash -c 'grafana-cli --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini admin reset-admin-password admin'
++ mktemp
+ local LAST_OUT=/tmp/tmp.aC5OxCAvnN
++ mktemp
+ local LAST_ERR=/tmp/tmp.FHRMsV0zUj
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'grafana-cli --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini admin reset-admin-password admin'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.aC5OxCAvnN
logger=settings t=2025-10-15T10:25:33.008198291Z level=info msg="Starting Grafana" version= commit= branch= compiled=1970-01-01T00:00:00Z
logger=settings t=2025-10-15T10:25:33.008335401Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini
logger=settings t=2025-10-15T10:25:33.008347071Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini
logger=settings t=2025-10-15T10:25:33.008352281Z level=info msg="Path Home" path=/usr/share/grafana
logger=settings t=2025-10-15T10:25:33.008356271Z level=info msg="Path Data" path=/srv/grafana
logger=settings t=2025-10-15T10:25:33.008360791Z level=info msg="Path Logs" path=/srv/logs
logger=settings t=2025-10-15T10:25:33.008366061Z level=info msg="Path Plugins" path=/srv/grafana/plugins
logger=settings t=2025-10-15T10:25:33.008370801Z level=info msg="Path Provisioning" path=/usr/share/grafana/conf/provisioning
logger=settings t=2025-10-15T10:25:33.008374981Z level=info msg="App mode production"
logger=sqlstore t=2025-10-15T10:25:33.008431441Z level=info msg="Connecting to DB" dbtype=postgres
logger=migrator t=2025-10-15T10:25:33.02299731Z level=info msg="Starting DB migrations"
logger=migrator t=2025-10-15T10:25:33.02622177Z level=info msg="migrations completed" performed=0 skipped=452 duration=292.8µs
logger=secrets t=2025-10-15T10:25:33.027363709Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1
logger=plugin.finder t=2025-10-15T10:25:33.057104217Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled
logger=plugin.signature.validator t=2025-10-15T10:25:33.177422587Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=pmm-qan-app-panel pluginDir=/srv/grafana/plugins/pmm-app/dist/pmm-qan
logger=plugin.signature.validator t=2025-10-15T10:25:33.177475307Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=grafana-polystat-panel pluginDir=/srv/grafana/plugins/grafana-polystat-panel
logger=plugin.signature.validator t=2025-10-15T10:25:33.177497637Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=pmm-app pluginDir=/srv/grafana/plugins/pmm-app/dist
logger=plugin.pmm-app t=2025-10-15T10:25:33.182871077Z level=warn msg="Included dashboard is missing a UID field"
logger=plugin.signature.validator t=2025-10-15T10:25:33.183062487Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=pmm-pt-summary-panel pluginDir=/srv/grafana/plugins/pmm-app/dist/pmm-pt-summary/panel
logger=plugin.signature.validator t=2025-10-15T10:25:33.183084237Z level=warn msg="Permitting unsigned plugin. This is not recommended" pluginID=pmm-pt-summary-datasource pluginDir=/srv/grafana/plugins/pmm-app/dist/pmm-pt-summary/datasource
logger=plugin.loader t=2025-10-15T10:25:33.183157537Z level=info msg="Plugin registered" pluginID=pmm-qan-app-panel
logger=plugin.loader t=2025-10-15T10:25:33.183170007Z level=info msg="Plugin registered" pluginID=camptocamp-prometheus-alertmanager-datasource
logger=plugin.loader t=2025-10-15T10:25:33.183176147Z level=info msg="Plugin registered" pluginID=jdbranham-diagram-panel
logger=plugin.loader t=2025-10-15T10:25:33.183182027Z level=info msg="Plugin registered" pluginID=natel-discrete-panel
logger=plugin.loader t=2025-10-15T10:25:33.183187197Z level=info msg="Plugin registered" pluginID=petrslavotinek-carpetplot-panel
logger=plugin.loader t=2025-10-15T10:25:33.183192217Z level=info msg="Plugin registered" pluginID=grafana-polystat-panel
logger=plugin.loader t=2025-10-15T10:25:33.183197557Z level=info msg="Plugin registered" pluginID=grafana-worldmap-panel
logger=plugin.loader t=2025-10-15T10:25:33.183202817Z level=info msg="Plugin registered" pluginID=pmm-app
logger=plugin.loader t=2025-10-15T10:25:33.183208407Z level=info msg="Plugin registered" pluginID=pmm-pt-summary-panel
logger=plugin.loader t=2025-10-15T10:25:33.183214897Z level=info msg="Plugin registered" pluginID=pmm-pt-summary-datasource
logger=plugin.loader t=2025-10-15T10:25:33.183222067Z level=info msg="Plugin registered" pluginID=grafana-clickhouse-datasource
logger=plugin.grafana-clickhouse-datasource t=2025-10-15T10:25:33.193135736Z level=warn msg="Plugin process is running with elevated privileges. This is not recommended"
logger=plugin.loader t=2025-10-15T10:25:33.193149076Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel

Admin password changed successfully ✔

+ cat /tmp/tmp.FHRMsV0zUj
+ rm /tmp/tmp.aC5OxCAvnN /tmp/tmp.FHRMsV0zUj
+ return 0
+ desc 'create PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC cluster
-----------------------------------------------------------------------------------

+ spinup_pxc monitoring /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/conf/monitoring.yml 3 120 /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/conf/secrets.yaml
+ local cluster=monitoring
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/conf/monitoring.yml
+ local size=3
+ local sleep=120
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/conf/secrets.yaml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/conf/secrets.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.7T4nZJDXnU
++ mktemp
+ local LAST_ERR=/tmp/tmp.1x8AIYwZds
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/conf/secrets.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7T4nZJDXnU
secret/my-cluster-secrets created
+ cat /tmp/tmp.1x8AIYwZds
+ rm /tmp/tmp.7T4nZJDXnU /tmp/tmp.1x8AIYwZds
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.monitoring-2-0-932~
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2213-6c08ea71#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.EYL36fXtUU
++ mktemp
+ local LAST_ERR=/tmp/tmp.pWbMGOaqPI
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EYL36fXtUU
deployment.apps/pxc-client created
+ cat /tmp/tmp.pWbMGOaqPI
+ rm /tmp/tmp.EYL36fXtUU /tmp/tmp.pWbMGOaqPI
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/conf/monitoring.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/conf/monitoring.yml
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.KdCzZD9DPT
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.monitoring-2-0-932~
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/conf/monitoring.yml
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2213-6c08ea71#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
++ mktemp
+ local LAST_ERR=/tmp/tmp.WajvMmTMzD
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.KdCzZD9DPT
perconaxtradbcluster.pxc.percona.com/monitoring created
+ cat /tmp/tmp.WajvMmTMzD
+ rm /tmp/tmp.KdCzZD9DPT /tmp/tmp.WajvMmTMzD
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy monitoring
++ local target_cluster=monitoring
+++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.eQ3G1rwGhg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.yrZ2eI8iTg
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.eQ3G1rwGhg
+++ cat /tmp/tmp.yrZ2eI8iTg
+++ rm /tmp/tmp.eQ3G1rwGhg /tmp/tmp.yrZ2eI8iTg
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo monitoring-haproxy
++ return
+ local proxy=monitoring-haproxy
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n monitoring-2-0-932
++ mktemp
+ local LAST_OUT=/tmp/tmp.Dlm4VQcPKy
++ mktemp
+ local LAST_ERR=/tmp/tmp.xdVyxu6zef
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n monitoring-2-0-932
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n monitoring-2-0-932
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n monitoring-2-0-932
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Dlm4VQcPKy
pod/monitoring-haproxy-0 condition met
pod/monitoring-pxc-0 condition met
+ cat /tmp/tmp.xdVyxu6zef
+ rm /tmp/tmp.Dlm4VQcPKy /tmp/tmp.xdVyxu6zef
+ return 0
+ wait_for_running monitoring-haproxy 1
+ local name=monitoring-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod monitoring-haproxy-0 480
+ local pod=monitoring-haproxy-0
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
++ echo monitoring-haproxy-0
+ local container=
+ set +o xtrace
pod/monitoring-haproxy-0 condition met
waiting for pod/monitoring-haproxy-0 to become Ready.Ok
+ wait_for_running monitoring-pxc 3
+ local name=monitoring-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod monitoring-pxc-0 480
+ local pod=monitoring-pxc-0
+ local max_retry=480
+ local ns=
++ echo monitoring-pxc-0
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/monitoring-pxc-0 condition met
waiting for pod/monitoring-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod monitoring-pxc-1 480
+ local pod=monitoring-pxc-1
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo monitoring-pxc-1
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/monitoring-pxc-1 condition met
waiting for pod/monitoring-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod monitoring-pxc-2 480
+ local pod=monitoring-pxc-2
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
++ echo monitoring-pxc-2
+ local container=pxc
+ set +o xtrace
pod/monitoring-pxc-2 condition met
waiting for pod/monitoring-pxc-2 to become Ready.Ok
+ sleep 120
++ kubectl get pxc monitoring -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bQOmEQ54ta
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iysi1Yx6pb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bQOmEQ54ta
++ cat /tmp/tmp.iysi1Yx6pb
++ rm /tmp/tmp.bQOmEQ54ta /tmp/tmp.iysi1Yx6pb
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h monitoring-haproxy -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h monitoring-haproxy -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZQfrJqrm5a
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lmkRnBEsxb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZQfrJqrm5a
++ cat /tmp/tmp.lmkRnBEsxb
++ rm /tmp/tmp.ZQfrJqrm5a /tmp/tmp.lmkRnBEsxb
++ return 0
+ client_pod=pxc-client-59944c5bbf-4tt89
+ wait_pod pxc-client-59944c5bbf-4tt89
+ local pod=pxc-client-59944c5bbf-4tt89
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-4tt89
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-4tt89 condition met
waiting for pod/pxc-client-59944c5bbf-4tt89 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h monitoring-haproxy -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h monitoring-haproxy -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JQ9NcQvz1W
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wb2YvhEFQa
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.JQ9NcQvz1W
++ cat /tmp/tmp.wb2YvhEFQa
++ rm /tmp/tmp.JQ9NcQvz1W /tmp/tmp.wb2YvhEFQa
++ return 0
+ client_pod=pxc-client-59944c5bbf-4tt89
+ wait_pod pxc-client-59944c5bbf-4tt89
+ local pod=pxc-client-59944c5bbf-4tt89
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-4tt89
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-4tt89 condition met
waiting for pod/pxc-client-59944c5bbf-4tt89 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KQnrLPTj0C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jVJmBOxOqm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KQnrLPTj0C
++ cat /tmp/tmp.jVJmBOxOqm
++ rm /tmp/tmp.KQnrLPTj0C /tmp/tmp.jVJmBOxOqm
++ return 0
+ client_pod=pxc-client-59944c5bbf-4tt89
+ wait_pod pxc-client-59944c5bbf-4tt89
+ local pod=pxc-client-59944c5bbf-4tt89
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-4tt89
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-4tt89 condition met
waiting for pod/pxc-client-59944c5bbf-4tt89 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.h65oP78O1Z/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/select-1.sql /tmp/tmp.h65oP78O1Z/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h monitoring-pxc-1.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h monitoring-pxc-1.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h monitoring-pxc-1.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h monitoring-pxc-1.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ov1zo56TQY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.een4lqsANe
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Ov1zo56TQY
++ cat /tmp/tmp.een4lqsANe
++ rm /tmp/tmp.Ov1zo56TQY /tmp/tmp.een4lqsANe
++ return 0
+ client_pod=pxc-client-59944c5bbf-4tt89
+ wait_pod pxc-client-59944c5bbf-4tt89
+ local pod=pxc-client-59944c5bbf-4tt89
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-4tt89
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-4tt89 condition met
waiting for pod/pxc-client-59944c5bbf-4tt89 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.h65oP78O1Z/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/select-1.sql /tmp/tmp.h65oP78O1Z/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h monitoring-pxc-2.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h monitoring-pxc-2.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h monitoring-pxc-2.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h monitoring-pxc-2.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8iO55rEY1S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MhQabzzy6X
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8iO55rEY1S
++ cat /tmp/tmp.MhQabzzy6X
++ rm /tmp/tmp.8iO55rEY1S /tmp/tmp.MhQabzzy6X
++ return 0
+ client_pod=pxc-client-59944c5bbf-4tt89
+ wait_pod pxc-client-59944c5bbf-4tt89
+ local pod=pxc-client-59944c5bbf-4tt89
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-4tt89
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-4tt89 condition met
waiting for pod/pxc-client-59944c5bbf-4tt89 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.h65oP78O1Z/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/select-1.sql /tmp/tmp.h65oP78O1Z/select-1.sql
++ is_keyring_plugin_in_use monitoring
++ local cluster=monitoring
++ kubectl_bin exec -it monitoring-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eRrCorKV6i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5wPJZGo7sM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it monitoring-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eRrCorKV6i
++ cat /tmp/tmp.5wPJZGo7sM
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.eRrCorKV6i /tmp/tmp.5wPJZGo7sM
++ return 0
+ '[' '' ']'
+ desc 'add PMM API key to secret'
+ set +o xtrace


-----------------------------------------------------------------------------------
add PMM API key to secret
-----------------------------------------------------------------------------------

++ jq .key
+++ get_service_endpoint monitoring-service
+++ local service=monitoring-service
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].hostname'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.CFG3U41hxI
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.PUSaPtGPrY
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.CFG3U41hxI
++++ cat /tmp/tmp.PUSaPtGPrY
++++ rm /tmp/tmp.CFG3U41hxI /tmp/tmp.PUSaPtGPrY
++++ return 0
+++ local hostname=null
+++ '[' -n null -a null '!=' null ']'
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].ip'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.f8AkX3R2yW
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.Rm1i7uIENo
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.f8AkX3R2yW
++++ cat /tmp/tmp.Rm1i7uIENo
++++ rm /tmp/tmp.f8AkX3R2yW /tmp/tmp.Rm1i7uIENo
++++ return 0
+++ local ip=34.61.206.88
+++ '[' -n 34.61.206.88 -a 34.61.206.88 '!=' null ']'
+++ echo 34.61.206.88
+++ return
++ curl --insecure -X POST -H 'Content-Type: application/json' -d '{"name":"operator", "role": "Admin"}' https://admin:admin@34.61.206.88/graph/api/auth/keys
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100   155  100   119  100    36    272     82 --:--:-- --:--:-- --:--:--   355
+ API_KEY='"eyJrIjoiWnp6NVhyT1YzU1dwajRGZjl0M25VcWZ4MDdXRHZYeDEiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="'
+ kubectl_bin patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": "eyJrIjoiWnp6NVhyT1YzU1dwajRGZjl0M25VcWZ4MDdXRHZYeDEiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.KuMeU56jia
++ mktemp
+ local LAST_ERR=/tmp/tmp.5Nkh34kQpX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": "eyJrIjoiWnp6NVhyT1YzU1dwajRGZjl0M25VcWZ4MDdXRHZYeDEiLCJuIjoib3BlcmF0b3IiLCJpZCI6MX0="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.KuMeU56jia
secret/my-cluster-secrets patched
+ cat /tmp/tmp.5Nkh34kQpX
+ rm /tmp/tmp.KuMeU56jia /tmp/tmp.5Nkh34kQpX
+ return 0
+ wait_for_generation sts/monitoring-pxc 2
+ local resource=sts/monitoring-pxc
+ local target_generation=2
+ echo 'Waiting for sts/monitoring-pxc to reach generation 2...'
Waiting for sts/monitoring-pxc to reach generation 2...
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=1
+ '[' 1 -eq 2 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 1. Waiting...'
Resource sts/monitoring-pxc is at generation 1. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=2
+ '[' 2 -eq 2 ']'
+ echo 'Resource sts/monitoring-pxc has reached generation 2.'
Resource sts/monitoring-pxc has reached generation 2.
+ break
+ wait_for_generation sts/monitoring-haproxy 2
+ local resource=sts/monitoring-haproxy
+ local target_generation=2
+ echo 'Waiting for sts/monitoring-haproxy to reach generation 2...'
Waiting for sts/monitoring-haproxy to reach generation 2...
+ true
++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}'
+ current_generation=2
+ '[' 2 -eq 2 ']'
+ echo 'Resource sts/monitoring-haproxy has reached generation 2.'
Resource sts/monitoring-haproxy has reached generation 2.
+ break
+ sleep 10
+ kubectl wait pod -l app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --for=condition=ready --timeout=600s
pod/monitoring-haproxy-0 condition met
pod/monitoring-haproxy-1 condition met
pod/monitoring-pxc-0 condition met
pod/monitoring-pxc-1 condition met
pod/monitoring-pxc-2 condition met
+ wait_cluster_consistency monitoring 3 2
+ local cluster_name=monitoring
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/monitoring to be ready'
waiting for pxc/monitoring to be ready++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.i1COnsDGbk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.obUBCD3N8K
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.i1COnsDGbk
++ cat /tmp/tmp.obUBCD3N8K
++ rm /tmp/tmp.i1COnsDGbk /tmp/tmp.obUBCD3N8K
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 0 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XbIZCZOskK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CX0DxCb9R7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XbIZCZOskK
++ cat /tmp/tmp.CX0DxCb9R7
++ rm /tmp/tmp.XbIZCZOskK /tmp/tmp.CX0DxCb9R7
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 1 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SvdVhFvqLS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2phyNajPNO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SvdVhFvqLS
++ cat /tmp/tmp.2phyNajPNO
++ rm /tmp/tmp.SvdVhFvqLS /tmp/tmp.2phyNajPNO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 2 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wVA77jvQnp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rcbbspEGSs
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wVA77jvQnp
++ cat /tmp/tmp.rcbbspEGSs
++ rm /tmp/tmp.wVA77jvQnp /tmp/tmp.rcbbspEGSs
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 3 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hPLFrV7PBP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IiQwd2URRM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hPLFrV7PBP
++ cat /tmp/tmp.IiQwd2URRM
++ rm /tmp/tmp.hPLFrV7PBP /tmp/tmp.IiQwd2URRM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 4 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SNCqBLbKvV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kkUQub99pO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SNCqBLbKvV
++ cat /tmp/tmp.kkUQub99pO
++ rm /tmp/tmp.SNCqBLbKvV /tmp/tmp.kkUQub99pO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 5 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DQItOo516k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HdLg2Ehfml
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DQItOo516k
++ cat /tmp/tmp.HdLg2Ehfml
++ rm /tmp/tmp.DQItOo516k /tmp/tmp.HdLg2Ehfml
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 6 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S4N5l0BJ2V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ct5K5dyv5D
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.S4N5l0BJ2V
++ cat /tmp/tmp.ct5K5dyv5D
++ rm /tmp/tmp.S4N5l0BJ2V /tmp/tmp.ct5K5dyv5D
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 7 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CvdH9Trxod
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ECySAcWYkZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.CvdH9Trxod
++ cat /tmp/tmp.ECySAcWYkZ
++ rm /tmp/tmp.CvdH9Trxod /tmp/tmp.ECySAcWYkZ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 8 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nazjHt1Rqz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KX54rKQUsV
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nazjHt1Rqz
++ cat /tmp/tmp.KX54rKQUsV
++ rm /tmp/tmp.nazjHt1Rqz /tmp/tmp.KX54rKQUsV
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 9 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rdRmcMq2aI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NxX8gFwEKB
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rdRmcMq2aI
++ cat /tmp/tmp.NxX8gFwEKB
++ rm /tmp/tmp.rdRmcMq2aI /tmp/tmp.NxX8gFwEKB
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 10 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FAfrrBdkPO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jFUy3LiO9v
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FAfrrBdkPO
++ cat /tmp/tmp.jFUy3LiO9v
++ rm /tmp/tmp.FAfrrBdkPO /tmp/tmp.jFUy3LiO9v
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 11 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1SiCXmee3x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eLXJQPnWB1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1SiCXmee3x
++ cat /tmp/tmp.eLXJQPnWB1
++ rm /tmp/tmp.1SiCXmee3x /tmp/tmp.eLXJQPnWB1
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 12 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k2ecOoj5TA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AEkbora0TA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.k2ecOoj5TA
++ cat /tmp/tmp.AEkbora0TA
++ rm /tmp/tmp.k2ecOoj5TA /tmp/tmp.AEkbora0TA
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 13 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y0PrzXRD8i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iVmtvWZE6D
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.y0PrzXRD8i
++ cat /tmp/tmp.iVmtvWZE6D
++ rm /tmp/tmp.y0PrzXRD8i /tmp/tmp.iVmtvWZE6D
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 14 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZpTA5wDCBP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AtzepSeKyB
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZpTA5wDCBP
++ cat /tmp/tmp.AtzepSeKyB
++ rm /tmp/tmp.ZpTA5wDCBP /tmp/tmp.AtzepSeKyB
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 15 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xkI8ZGlLe2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q8WZ7RSCMp
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xkI8ZGlLe2
++ cat /tmp/tmp.q8WZ7RSCMp
++ rm /tmp/tmp.xkI8ZGlLe2 /tmp/tmp.q8WZ7RSCMp
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8jqMYA0Tw7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HteBae9pF5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8jqMYA0Tw7
++ cat /tmp/tmp.HteBae9pF5
++ rm /tmp/tmp.8jqMYA0Tw7 /tmp/tmp.HteBae9pF5
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine monitoring
+++ local cluster_name=monitoring
++++ get_proxy monitoring
++++ local target_cluster=monitoring
+++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.HovABAY2yw
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.WYDr8uUQBN
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.HovABAY2yw
+++++ cat /tmp/tmp.WYDr8uUQBN
+++++ rm /tmp/tmp.HovABAY2yw /tmp/tmp.WYDr8uUQBN
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo monitoring-haproxy
++++ return
+++ local cluster_proxy=monitoring-haproxy
+++ echo haproxy
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RzQvzTgmzP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.V7x4zjrHDk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RzQvzTgmzP
++ cat /tmp/tmp.V7x4zjrHDk
++ rm /tmp/tmp.RzQvzTgmzP /tmp/tmp.V7x4zjrHDk
++ return 0
+ [[ 2 == \2 ]]
+ echo

+ compare_kubectl statefulset/monitoring-pxc -no-prefix
+ local resource=statefulset/monitoring-pxc
+ local postfix=-no-prefix
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix.yml
+ local new_result=/tmp/tmp.h65oP78O1Z/statefulset_monitoring-pxc.yml
+ desc 'compare statefulset/monitoring-pxc--no-prefix'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/monitoring-pxc--no-prefix
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-80.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ version_gt 1.33
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.33'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k129.yml ']'
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.27'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127.yml ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127.yml
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127-oc.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127-k129-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127-eks.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127-aks.yml ']'
+ kubectl_bin get -o yaml statefulset/monitoring-pxc
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-932", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.spec.crVersion) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.5qNGqujA8g
++ mktemp
+ local LAST_ERR=/tmp/tmp.c4X9AsstFy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5qNGqujA8g
+ cat /tmp/tmp.c4X9AsstFy
+ rm /tmp/tmp.5qNGqujA8g /tmp/tmp.c4X9AsstFy
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-no-prefix-k127.yml /tmp/tmp.h65oP78O1Z/statefulset_monitoring-pxc.yml
+ compare_kubectl statefulset/monitoring-haproxy -no-prefix
+ local resource=statefulset/monitoring-haproxy
+ local postfix=-no-prefix
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix.yml
+ local new_result=/tmp/tmp.h65oP78O1Z/statefulset_monitoring-haproxy.yml
+ desc 'compare statefulset/monitoring-haproxy--no-prefix'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/monitoring-haproxy--no-prefix
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-80.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ version_gt 1.33
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.31 >= 1.33'
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.31 >= 1.29'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k129.yml ']'
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.31 >= 1.27'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127.yml ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127.yml
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127-oc.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127-k129-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127-eks.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127-aks.yml ']'
+ kubectl_bin get -o yaml statefulset/monitoring-haproxy
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-932", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.spec.crVersion) |
++ mktemp
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.rRGqHVFWgF
++ mktemp
+ local LAST_ERR=/tmp/tmp.uRHxqTOaNH
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-haproxy
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rRGqHVFWgF
+ cat /tmp/tmp.uRHxqTOaNH
+ rm /tmp/tmp.rRGqHVFWgF /tmp/tmp.uRHxqTOaNH
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-no-prefix-k127.yml /tmp/tmp.h65oP78O1Z/statefulset_monitoring-haproxy.yml
+ desc 'apply my-env-var-secrets to add PMM_PREFIX'
+ set +o xtrace


-----------------------------------------------------------------------------------
apply my-env-var-secrets to add PMM_PREFIX
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/conf/envsecrets.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.RUh7knuAHc
++ mktemp
+ local LAST_ERR=/tmp/tmp.QkFYUf8VB2
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/conf/envsecrets.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.RUh7knuAHc
secret/my-env-var-secrets created
+ cat /tmp/tmp.QkFYUf8VB2
+ rm /tmp/tmp.RUh7knuAHc /tmp/tmp.QkFYUf8VB2
+ return 0
+ wait_for_generation sts/monitoring-pxc 3
+ local resource=sts/monitoring-pxc
+ local target_generation=3
+ echo 'Waiting for sts/monitoring-pxc to reach generation 3...'
Waiting for sts/monitoring-pxc to reach generation 3...
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=2
+ '[' 2 -eq 3 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 2. Waiting...'
Resource sts/monitoring-pxc is at generation 2. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 3 ']'
+ echo 'Resource sts/monitoring-pxc has reached generation 3.'
Resource sts/monitoring-pxc has reached generation 3.
+ break
+ wait_for_generation sts/monitoring-haproxy 3
+ local resource=sts/monitoring-haproxy
+ local target_generation=3
+ echo 'Waiting for sts/monitoring-haproxy to reach generation 3...'
Waiting for sts/monitoring-haproxy to reach generation 3...
+ true
++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 3 ']'
+ echo 'Resource sts/monitoring-haproxy has reached generation 3.'
Resource sts/monitoring-haproxy has reached generation 3.
+ break
+ desc 'add new PMM API key to secret'
+ set +o xtrace


-----------------------------------------------------------------------------------
add new PMM API key to secret
-----------------------------------------------------------------------------------

++ jq .key
+++ get_service_endpoint monitoring-service
+++ local service=monitoring-service
++++ jq '.status.loadBalancer.ingress[].hostname'
++++ kubectl_bin get service/monitoring-service -o json
+++++ mktemp
++++ sed -e 's/^"//; s/"$//;'
++++ local LAST_OUT=/tmp/tmp.929GVkec6o
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.KVSk66UoaC
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.929GVkec6o
++++ cat /tmp/tmp.KVSk66UoaC
++++ rm /tmp/tmp.929GVkec6o /tmp/tmp.KVSk66UoaC
++++ return 0
+++ local hostname=null
+++ '[' -n null -a null '!=' null ']'
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].ip'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.mBXXmfn1GO
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.14N9SWvma1
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.mBXXmfn1GO
++++ cat /tmp/tmp.14N9SWvma1
++++ rm /tmp/tmp.mBXXmfn1GO /tmp/tmp.14N9SWvma1
++++ return 0
+++ local ip=34.61.206.88
+++ '[' -n 34.61.206.88 -a 34.61.206.88 '!=' null ']'
+++ echo 34.61.206.88
+++ return
++ curl --insecure -X POST -H 'Content-Type: application/json' -d '{"name":"operator-new", "role": "Admin"}' https://admin:admin@34.61.206.88/graph/api/auth/keys
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100   167  100   127  100    40    288     90 --:--:-- --:--:-- --:--:--   379
+ API_KEY_NEW='"eyJrIjoiQmJGbzVna2d0U1NtVEIxcHYyYW50c2h1WXVtNkN0ZXgiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9"'
+ kubectl_bin patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": "eyJrIjoiQmJGbzVna2d0U1NtVEIxcHYyYW50c2h1WXVtNkN0ZXgiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9"}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ESFeDd5ONc
++ mktemp
+ local LAST_ERR=/tmp/tmp.E5WURyQCNV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": "eyJrIjoiQmJGbzVna2d0U1NtVEIxcHYyYW50c2h1WXVtNkN0ZXgiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9"}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ESFeDd5ONc
secret/my-cluster-secrets patched
+ cat /tmp/tmp.E5WURyQCNV
+ rm /tmp/tmp.ESFeDd5ONc /tmp/tmp.E5WURyQCNV
+ return 0
+ desc 'delete old PMM key'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete old PMM key
-----------------------------------------------------------------------------------

++ jq '.[] | select( .name == "operator").id'
+++ get_service_endpoint monitoring-service
+++ local service=monitoring-service
++++ kubectl_bin get service/monitoring-service -o json
++++ sed -e 's/^"//; s/"$//;'
++++ jq '.status.loadBalancer.ingress[].hostname'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.NhIEg8jzDX
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.4l2UMwwh4W
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.NhIEg8jzDX
++++ cat /tmp/tmp.4l2UMwwh4W
++++ rm /tmp/tmp.NhIEg8jzDX /tmp/tmp.4l2UMwwh4W
++++ return 0
+++ local hostname=null
+++ '[' -n null -a null '!=' null ']'
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].ip'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.5fvsHTjCSx
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.dK1e6Hvs4m
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.5fvsHTjCSx
++++ cat /tmp/tmp.dK1e6Hvs4m
++++ rm /tmp/tmp.5fvsHTjCSx /tmp/tmp.dK1e6Hvs4m
++++ return 0
+++ local ip=34.61.206.88
+++ '[' -n 34.61.206.88 -a 34.61.206.88 '!=' null ']'
+++ echo 34.61.206.88
+++ return
++ curl --insecure -X GET https://admin:admin@34.61.206.88/graph/api/auth/keys
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100   592  100   592    0     0   1364      0 --:--:-- --:--:-- --:--:--  1367
+ ID_API_KEY_OLD=6
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ sed -e 's/^"//; s/"$//;'
+++ jq '.status.loadBalancer.ingress[].hostname'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.trFFAnYl9G
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NpCs8kNVTs
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.trFFAnYl9G
+++ cat /tmp/tmp.NpCs8kNVTs
+++ rm /tmp/tmp.trFFAnYl9G /tmp/tmp.NpCs8kNVTs
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
++++ mktemp
+++ jq '.status.loadBalancer.ingress[].ip'
+++ local LAST_OUT=/tmp/tmp.auO7dq9EIY
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.sfuURDlBeL
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.auO7dq9EIY
+++ cat /tmp/tmp.sfuURDlBeL
+++ rm /tmp/tmp.auO7dq9EIY /tmp/tmp.sfuURDlBeL
+++ return 0
++ local ip=34.61.206.88
++ '[' -n 34.61.206.88 -a 34.61.206.88 '!=' null ']'
++ echo 34.61.206.88
++ return
+ curl --insecure -X DELETE https://admin:admin@34.61.206.88/graph/api/auth/keys/6
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100    29  100    29    0     0     66      0 --:--:-- --:--:-- --:--:--    66
{"message":"API key deleted"}+ wait_for_generation sts/monitoring-pxc 4
+ local resource=sts/monitoring-pxc
+ local target_generation=4
+ echo 'Waiting for sts/monitoring-pxc to reach generation 4...'
Waiting for sts/monitoring-pxc to reach generation 4...
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=3
+ '[' 3 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc is at generation 3. Waiting...'
Resource sts/monitoring-pxc is at generation 3. Waiting...
+ sleep 5
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=4
+ '[' 4 -eq 4 ']'
+ echo 'Resource sts/monitoring-pxc has reached generation 4.'
Resource sts/monitoring-pxc has reached generation 4.
+ break
+ wait_for_generation sts/monitoring-haproxy 4
+ local resource=sts/monitoring-haproxy
+ local target_generation=4
+ echo 'Waiting for sts/monitoring-haproxy to reach generation 4...'
Waiting for sts/monitoring-haproxy to reach generation 4...
+ true
++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}'
+ current_generation=4
+ '[' 4 -eq 4 ']'
+ echo 'Resource sts/monitoring-haproxy has reached generation 4.'
Resource sts/monitoring-haproxy has reached generation 4.
+ break
+ sleep 10
+ kubectl wait pod -l app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --for=condition=ready --timeout=600s
pod/monitoring-haproxy-0 condition met
pod/monitoring-haproxy-1 condition met
pod/monitoring-pxc-0 condition met
pod/monitoring-pxc-1 condition met
pod/monitoring-pxc-2 condition met
+ desc 'check if pmm-client container enabled'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if pmm-client container enabled
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/monitoring-pxc
+ local resource=statefulset/monitoring-pxc
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc.yml
+ local new_result=/tmp/tmp.h65oP78O1Z/statefulset_monitoring-pxc.yml
+ desc 'compare statefulset/monitoring-pxc-'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/monitoring-pxc-
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-80.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ version_gt 1.33
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.33'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k129.yml ']'
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.31 >= 1.27'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127.yml ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127.yml
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127-oc.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127-k129-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127-eks.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127-aks.yml ']'
+ kubectl_bin get -o yaml statefulset/monitoring-pxc
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-932", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.spec.crVersion) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.vcWLmNORTf
++ mktemp
+ local LAST_ERR=/tmp/tmp.J2mDWmAuhE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vcWLmNORTf
+ cat /tmp/tmp.J2mDWmAuhE
+ rm /tmp/tmp.vcWLmNORTf /tmp/tmp.J2mDWmAuhE
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-pxc-k127.yml /tmp/tmp.h65oP78O1Z/statefulset_monitoring-pxc.yml
+ compare_kubectl statefulset/monitoring-haproxy
+ local resource=statefulset/monitoring-haproxy
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy.yml
+ local new_result=/tmp/tmp.h65oP78O1Z/statefulset_monitoring-haproxy.yml
+ desc 'compare statefulset/monitoring-haproxy-'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/monitoring-haproxy-
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-80.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ version_gt 1.33
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.33'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.31 >= 1.29'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k129.yml ']'
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.31 >= 1.27'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127.yml ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127.yml
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127-oc.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.31 >= 1.29'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127-k129-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127-eks.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127-aks.yml ']'
+ kubectl_bin get -o yaml statefulset/monitoring-haproxy
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("monitoring-2-0-932", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.spec.crVersion) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Eh1ezClHxX
++ mktemp
+ local LAST_ERR=/tmp/tmp.yoBIGS68BM
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-haproxy
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Eh1ezClHxX
+ cat /tmp/tmp.yoBIGS68BM
+ rm /tmp/tmp.Eh1ezClHxX /tmp/tmp.yoBIGS68BM
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/statefulset_monitoring-haproxy-k127.yml /tmp/tmp.h65oP78O1Z/statefulset_monitoring-haproxy.yml
+ desc 'verify clients agents statuses'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify clients agents statuses
-----------------------------------------------------------------------------------

+ sleep 300
++ getSecretData my-cluster-secrets pmmserverkey
++ local secretName=my-cluster-secrets
++ local dataKey=pmmserverkey
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.pmmserverkey}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dfDviJAgwE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CgBerMlclG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.pmmserverkey}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.dfDviJAgwE
++ cat /tmp/tmp.CgBerMlclG
++ rm /tmp/tmp.dfDviJAgwE /tmp/tmp.CgBerMlclG
++ return 0
+ API_KEY=eyJrIjoiQmJGbzVna2d0U1NtVEIxcHYyYW50c2h1WXVtNkN0ZXgiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9
+++ get_service_endpoint monitoring-service
+++ local service=monitoring-service
++++ kubectl_bin get service/monitoring-service -o json
++++ jq '.status.loadBalancer.ingress[].hostname'
++++ sed -e 's/^"//; s/"$//;'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.SfbtaWQJmM
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.oOrzGFCuoB
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.SfbtaWQJmM
++++ cat /tmp/tmp.oOrzGFCuoB
++++ rm /tmp/tmp.SfbtaWQJmM /tmp/tmp.oOrzGFCuoB
++++ return 0
+++ local hostname=null
+++ '[' -n null -a null '!=' null ']'
++++ jq '.status.loadBalancer.ingress[].ip'
++++ sed -e 's/^"//; s/"$//;'
++++ kubectl_bin get service/monitoring-service -o json
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.aVCR4SqguM
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.RszS1yJmFk
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.aVCR4SqguM
++++ cat /tmp/tmp.RszS1yJmFk
++++ rm /tmp/tmp.aVCR4SqguM /tmp/tmp.RszS1yJmFk
++++ return 0
+++ local ip=34.61.206.88
+++ '[' -n 34.61.206.88 -a 34.61.206.88 '!=' null ']'
+++ echo 34.61.206.88
+++ return
++ get_mgmnt_service_list eyJrIjoiQmJGbzVna2d0U1NtVEIxcHYyYW50c2h1WXVtNkN0ZXgiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9 34.61.206.88 monitoring-2-0-932
++ local api_key=eyJrIjoiQmJGbzVna2d0U1NtVEIxcHYyYW50c2h1WXVtNkN0ZXgiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9
++ local endpoint=34.61.206.88
++ local namespace=monitoring-2-0-932
++ jq 'walk(if type == "array" then sort_by(.agent_type) else . end)'
++ jq 'walk(if type=="object" then with_entries(select(.key | test("service_id|node_id|agent_id|created_at|updated_at") | not)) else . end)'
++ curl -s -k -H 'Authorization: Bearer eyJrIjoiQmJGbzVna2d0U1NtVEIxcHYyYW50c2h1WXVtNkN0ZXgiLCJuIjoib3BlcmF0b3ItbmV3IiwiaWQiOjF9' -X POST https://34.61.206.88/v1/management/Service/List
++ /usr/bin/sed -i s/monitoring-2-0-932-//g /tmp/tmp.h65oP78O1Z/active_pmm_agents.json
++ cat /tmp/tmp.h65oP78O1Z/active_pmm_agents.json
++ jq '.services | sort_by(.node_name)'
++ echo /tmp/tmp.h65oP78O1Z/active_pmm_agents_sorted.json
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2213/e2e-tests/monitoring-2-0/compare/agents-list.json /tmp/tmp.h65oP78O1Z/active_pmm_agents_sorted.json
+ desc 'check mysql metrics'
+ set +o xtrace


-----------------------------------------------------------------------------------
check mysql metrics
-----------------------------------------------------------------------------------

+ get_metric_values node_boot_time_seconds pxc-prefix-monitoring-2-0-932-monitoring-pxc-0 admin:admin
+ local metric=node_boot_time_seconds
+ local instance=pxc-prefix-monitoring-2-0-932-monitoring-pxc-0
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1760524968
++ /usr/bin/date -u +%s
+ local end=1760525028
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.AXgyRrJcpI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.EYkI3pfemz
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.AXgyRrJcpI
+++ cat /tmp/tmp.EYkI3pfemz
+++ rm /tmp/tmp.AXgyRrJcpI /tmp/tmp.EYkI3pfemz
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ jq '.status.loadBalancer.ingress[].ip'
+++ kubectl_bin get service/monitoring-service -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5TKg8JZdDx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.d9ljyxclnq
+++ local exit_status=0
++++ seq 0 2
+++ sed -e 's/^"//; s/"$//;'
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.5TKg8JZdDx
+++ cat /tmp/tmp.d9ljyxclnq
+++ rm /tmp/tmp.5TKg8JZdDx /tmp/tmp.d9ljyxclnq
+++ return 0
++ local ip=34.61.206.88
++ '[' -n 34.61.206.88 -a 34.61.206.88 '!=' null ']'
++ echo 34.61.206.88
++ return
+ local endpoint=34.61.206.88
++ jq '.data.result[0]'
++ curl -s -k 'https://admin:admin@34.61.206.88/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-932-monitoring-pxc-0%22%7d%20or%20node_boot_time_seconds%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-932-monitoring-pxc-0%22%7D%29&start=1760524968&end=1760525028&step=60'
+ local 'result={
  "metric": {},
  "values": [
    [
      1760524968,
      "1760520891"
    ],
    [
      1760525028,
      "1760520891"
    ]
  ]
}'
+ '[' '{
  "metric": {},
  "values": [
    [
      1760524968,
      "1760520891"
    ],
    [
      1760525028,
      "1760520891"
    ]
  ]
}' = null ']'
+ grep '^"[0-9]'
+ echo -n '{
  "metric": {},
  "values": [
    [
      1760524968,
      "1760520891"
    ],
    [
      1760525028,
      "1760520891"
    ]
  ]
+ jq '.values[][1]'
}'
"1760520891"
"1760520891"
+ get_metric_values mysql_global_status_uptime pxc-prefix-monitoring-2-0-932-monitoring-pxc-0 admin:admin
+ local metric=mysql_global_status_uptime
+ local instance=pxc-prefix-monitoring-2-0-932-monitoring-pxc-0
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1760524971
++ /usr/bin/date -u +%s
+ local end=1760525031
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ kubectl_bin get service/monitoring-service -o json
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5FtAz52NNq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kTI2j9ouL0
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.5FtAz52NNq
+++ cat /tmp/tmp.kTI2j9ouL0
+++ rm /tmp/tmp.5FtAz52NNq /tmp/tmp.kTI2j9ouL0
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ jq '.status.loadBalancer.ingress[].ip'
+++ kubectl_bin get service/monitoring-service -o json
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WjevCE3wxP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bKpQy0B9So
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.WjevCE3wxP
+++ cat /tmp/tmp.bKpQy0B9So
+++ rm /tmp/tmp.WjevCE3wxP /tmp/tmp.bKpQy0B9So
+++ return 0
++ local ip=34.61.206.88
++ '[' -n 34.61.206.88 -a 34.61.206.88 '!=' null ']'
++ echo 34.61.206.88
++ return
+ local endpoint=34.61.206.88
++ curl -s -k 'https://admin:admin@34.61.206.88/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28mysql_global_status_uptime%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-932-monitoring-pxc-0%22%7d%20or%20mysql_global_status_uptime%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-932-monitoring-pxc-0%22%7D%29&start=1760524971&end=1760525031&step=60'
++ jq '.data.result[0]'
+ local 'result={
  "metric": {},
  "values": [
    [
      1760524971,
      "167"
    ],
    [
      1760525031,
      "227"
    ]
  ]
}'
+ '[' '{
  "metric": {},
  "values": [
    [
      1760524971,
      "167"
    ],
    [
      1760525031,
      "227"
    ]
  ]
}' = null ']'
+ echo -n '{
  "metric": {},
  "values": [
    [
      1760524971,
      "167"
    ],
    [
      1760525031,
      "227"
    ]
  ]
}'
+ grep '^"[0-9]'
+ jq '.values[][1]'
"167"
"227"
+ desc 'check haproxy metrics'
+ set +o xtrace


-----------------------------------------------------------------------------------
check haproxy metrics
-----------------------------------------------------------------------------------

+ get_metric_values haproxy_backend_status pxc-prefix-monitoring-2-0-932-monitoring-haproxy-0 admin:admin
+ local metric=haproxy_backend_status
+ local instance=pxc-prefix-monitoring-2-0-932-monitoring-haproxy-0
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1760524974
++ /usr/bin/date -u +%s
+ local end=1760525034
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ kubectl_bin get service/monitoring-service -o json
+++ sed -e 's/^"//; s/"$//;'
+++ jq '.status.loadBalancer.ingress[].hostname'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YuP5qGHwpr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.PcDreBWkbI
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.YuP5qGHwpr
+++ cat /tmp/tmp.PcDreBWkbI
+++ rm /tmp/tmp.YuP5qGHwpr /tmp/tmp.PcDreBWkbI
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
++++ mktemp
+++ sed -e 's/^"//; s/"$//;'
+++ local LAST_OUT=/tmp/tmp.VxBUyca49h
+++ jq '.status.loadBalancer.ingress[].ip'
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Xh1SUT8OMP
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.VxBUyca49h
+++ cat /tmp/tmp.Xh1SUT8OMP
+++ rm /tmp/tmp.VxBUyca49h /tmp/tmp.Xh1SUT8OMP
+++ return 0
++ local ip=34.61.206.88
++ '[' -n 34.61.206.88 -a 34.61.206.88 '!=' null ']'
++ echo 34.61.206.88
++ return
+ local endpoint=34.61.206.88
++ curl -s -k 'https://admin:admin@34.61.206.88/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28haproxy_backend_status%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-932-monitoring-haproxy-0%22%7d%20or%20haproxy_backend_status%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-932-monitoring-haproxy-0%22%7D%29&start=1760524974&end=1760525034&step=60'
++ jq '.data.result[0]'
+ local 'result={
  "metric": {},
  "values": [
    [
      1760524974,
      "0"
    ],
    [
      1760525034,
      "0"
    ]
  ]
}'
+ '[' '{
  "metric": {},
  "values": [
    [
      1760524974,
      "0"
    ],
    [
      1760525034,
      "0"
    ]
  ]
}' = null ']'
+ jq '.values[][1]'
+ grep '^"[0-9]'
+ echo -n '{
  "metric": {},
  "values": [
    [
      1760524974,
      "0"
    ],
    [
      1760525034,
      "0"
    ]
  ]
}'
"0"
"0"
+ get_metric_values haproxy_backend_active_servers pxc-prefix-monitoring-2-0-932-monitoring-haproxy-0 admin:admin
+ local metric=haproxy_backend_active_servers
+ local instance=pxc-prefix-monitoring-2-0-932-monitoring-haproxy-0
+ local user_pass=admin:admin
++ /usr/bin/date -u +%s -d '-1 minute'
+ local start=1760524977
++ /usr/bin/date -u +%s
+ local end=1760525037
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ sed -e 's/^"//; s/"$//;'
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ kubectl_bin get service/monitoring-service -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yUuQp4YYNI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4xfifVksXh
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.yUuQp4YYNI
+++ cat /tmp/tmp.4xfifVksXh
+++ rm /tmp/tmp.yUuQp4YYNI /tmp/tmp.4xfifVksXh
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ sed -e 's/^"//; s/"$//;'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.U4vIFGzF6m
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xH6dijrj8j
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.U4vIFGzF6m
+++ cat /tmp/tmp.xH6dijrj8j
+++ rm /tmp/tmp.U4vIFGzF6m /tmp/tmp.xH6dijrj8j
+++ return 0
++ local ip=34.61.206.88
++ '[' -n 34.61.206.88 -a 34.61.206.88 '!=' null ']'
++ echo 34.61.206.88
++ return
+ local endpoint=34.61.206.88
++ jq '.data.result[0]'
++ curl -s -k 'https://admin:admin@34.61.206.88/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28haproxy_backend_active_servers%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-932-monitoring-haproxy-0%22%7d%20or%20haproxy_backend_active_servers%7Bnode_name%3D%7E%22pxc-prefix-monitoring-2-0-932-monitoring-haproxy-0%22%7D%29&start=1760524977&end=1760525037&step=60'
+ local 'result={
  "metric": {},
  "values": [
    [
      1760524977,
      "1"
    ],
    [
      1760525037,
      "1"
    ]
  ]
}'
+ '[' '{
  "metric": {},
  "values": [
    [
      1760524977,
      "1"
    ],
    [
      1760525037,
      "1"
    ]
  ]
}' = null ']'
+ jq '.values[][1]'
+ grep '^"[0-9]'
+ echo -n '{
  "metric": {},
  "values": [
    [
      1760524977,
      "1"
    ],
    [
      1760525037,
      "1"
    ]
  ]
}'
"1"
"1"
+ desc 'check QAN data'
+ set +o xtrace


-----------------------------------------------------------------------------------
check QAN data
-----------------------------------------------------------------------------------

+ get_qan20_values monitoring-pxc-0 admin:admin
+ local instance=monitoring-pxc-0
+ local user_pass=admin:admin
++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S -d '-30 minute'
+ local start=2025-10-15T10:14:02
++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S
+ local end=2025-10-15T10:44:02
++ get_service_endpoint monitoring-service
++ local service=monitoring-service
+++ sed -e 's/^"//; s/"$//;'
+++ jq '.status.loadBalancer.ingress[].hostname'
+++ kubectl_bin get service/monitoring-service -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DNXuBRKByS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rQXIOmdGf7
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.DNXuBRKByS
+++ cat /tmp/tmp.rQXIOmdGf7
+++ rm /tmp/tmp.DNXuBRKByS /tmp/tmp.rQXIOmdGf7
+++ return 0
++ local hostname=null
++ '[' -n null -a null '!=' null ']'
+++ kubectl_bin get service/monitoring-service -o json
+++ jq '.status.loadBalancer.ingress[].ip'
+++ sed -e 's/^"//; s/"$//;'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.l6e0CrFbYG
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.j3AUiCMGCX
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get service/monitoring-service -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.l6e0CrFbYG
+++ cat /tmp/tmp.j3AUiCMGCX
+++ rm /tmp/tmp.l6e0CrFbYG /tmp/tmp.j3AUiCMGCX
+++ return 0
++ local ip=34.61.206.88
++ '[' -n 34.61.206.88 -a 34.61.206.88 '!=' null ']'
++ echo 34.61.206.88
++ return
+ local endpoint=34.61.206.88
+ cat
++ /usr/bin/date -u -d '-12 hour' +%Y-%m-%dT%H:%M:%S%:z
++ /usr/bin/date -u +%Y-%m-%dT%H:%M:%S%:z
+ jq '.rows[].fingerprint'
+ curl -s -k -XPOST -d @payload.json https://admin:admin@34.61.206.88/v0/qan/GetReport
null
+ rm -f payload.json
+ nodeList=($(get_node_id_from_pmm))
++ get_node_id_from_pmm
++ nodeList=()
++ local -a nodeList
+++ kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns=NAME:.metadata.name
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9tGL2tevIW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.LngYMFTlVu
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns=NAME:.metadata.name
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.9tGL2tevIW
+++ cat /tmp/tmp.LngYMFTlVu
+++ rm /tmp/tmp.9tGL2tevIW /tmp/tmp.LngYMFTlVu
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ jq -r .pmm_agent_status.node_id
+++ kubectl_bin exec -n monitoring-2-0-932 monitoring-pxc-0 -c pmm-client -- pmm-admin status --json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.sticEa1qSW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vF7P2FlbjH
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-932 monitoring-pxc-0 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.sticEa1qSW
+++ cat /tmp/tmp.vF7P2FlbjH
+++ rm /tmp/tmp.sticEa1qSW /tmp/tmp.vF7P2FlbjH
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ kubectl_bin exec -n monitoring-2-0-932 monitoring-pxc-1 -c pmm-client -- pmm-admin status --json
+++ jq -r .pmm_agent_status.node_id
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ukPeI8zeQl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3NJ1ICcoEt
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-932 monitoring-pxc-1 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.ukPeI8zeQl
+++ cat /tmp/tmp.3NJ1ICcoEt
+++ rm /tmp/tmp.ukPeI8zeQl /tmp/tmp.3NJ1ICcoEt
+++ return 0
++ for instance in '$(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='\''NAME:.metadata.name'\'')'
++ nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
+++ jq -r .pmm_agent_status.node_id
+++ kubectl_bin exec -n monitoring-2-0-932 monitoring-pxc-2 -c pmm-client -- pmm-admin status --json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ht4LhCX8XP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.RN2tOR6gzk
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-932 monitoring-pxc-2 -c pmm-client -- pmm-admin status --json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.ht4LhCX8XP
+++ cat /tmp/tmp.RN2tOR6gzk
+++ rm /tmp/tmp.ht4LhCX8XP /tmp/tmp.RN2tOR6gzk
+++ return 0
++ echo /node_id/0c18f111-be34-4729-8701-2d53bd7f9341 /node_id/a4ffb25e-4a64-4317-acf9-6f0ae377e62a /node_id/7ac820ba-94f1-4a9f-9d8d-3e1a07ca929d
+ nodeList_from_pmm=($(does_node_id_exists "${nodeList[@]}"))
++ does_node_id_exists /node_id/0c18f111-be34-4729-8701-2d53bd7f9341 /node_id/a4ffb25e-4a64-4317-acf9-6f0ae377e62a /node_id/7ac820ba-94f1-4a9f-9d8d-3e1a07ca929d
++ nodeList=("$@")
++ local -a nodeList
++ nodeList_from_pmm=()
++ local -a nodeList_from_pmm
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ grep -q NotFound
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
+++ grep /node_id/0c18f111-be34-4729-8701-2d53bd7f9341
+++ awk '{print $4}'
+++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.ytykeoaUcH
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.WKVUZfFhEq
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.ytykeoaUcH
+++++ cat /tmp/tmp.WKVUZfFhEq
+++++ rm /tmp/tmp.ytykeoaUcH /tmp/tmp.WKVUZfFhEq
+++++ return 0
++++ '[' LoadBalancer = ClusterIP ']'
++++ egrep -q 'hostname|ip'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.iSr0W7PtgK
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.4XalQ9G9GX
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.iSr0W7PtgK
++++ cat /tmp/tmp.4XalQ9G9GX
++++ rm /tmp/tmp.iSr0W7PtgK /tmp/tmp.4XalQ9G9GX
++++ return 0
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.vo3VFkjmNq
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.07JRIm05NM
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.vo3VFkjmNq
++++ cat /tmp/tmp.07JRIm05NM
++++ rm /tmp/tmp.vo3VFkjmNq /tmp/tmp.07JRIm05NM
++++ return 0
+++ kubectl_bin exec -n monitoring-2-0-932 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.61.206.88/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jqV8vLnlJj
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1mNvikO2sp
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-932 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.61.206.88/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.jqV8vLnlJj
+++ cat /tmp/tmp.1mNvikO2sp
+++ rm /tmp/tmp.jqV8vLnlJj /tmp/tmp.1mNvikO2sp
+++ return 0
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/a4ffb25e-4a64-4317-acf9-6f0ae377e62a
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
+++ awk '{print $4}'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
++++ grep -q NotFound
+++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.qBfhVaZmWs
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.SdzEaii3ZL
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.qBfhVaZmWs
+++++ cat /tmp/tmp.SdzEaii3ZL
+++++ rm /tmp/tmp.qBfhVaZmWs /tmp/tmp.SdzEaii3ZL
+++++ return 0
++++ '[' LoadBalancer = ClusterIP ']'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}'
++++ egrep -q 'hostname|ip'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.QkvMYKskaV
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.kYtEGQBNkM
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.QkvMYKskaV
++++ cat /tmp/tmp.kYtEGQBNkM
++++ rm /tmp/tmp.QkvMYKskaV /tmp/tmp.kYtEGQBNkM
++++ return 0
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.E49PY8bCcF
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.e6eCqfDC5S
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.E49PY8bCcF
++++ cat /tmp/tmp.e6eCqfDC5S
++++ rm /tmp/tmp.E49PY8bCcF /tmp/tmp.e6eCqfDC5S
++++ return 0
+++ kubectl_bin exec -n monitoring-2-0-932 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.61.206.88/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MbC9RkVtdY
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6vYVzFkW0J
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-932 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.61.206.88/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.MbC9RkVtdY
+++ cat /tmp/tmp.6vYVzFkW0J
+++ rm /tmp/tmp.MbC9RkVtdY /tmp/tmp.6vYVzFkW0J
+++ return 0
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/7ac820ba-94f1-4a9f-9d8d-3e1a07ca929d
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ grep -q NotFound
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
+++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.SBRMukzxXj
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.sc57fKBCP5
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.SBRMukzxXj
+++++ cat /tmp/tmp.sc57fKBCP5
+++++ rm /tmp/tmp.SBRMukzxXj /tmp/tmp.sc57fKBCP5
+++++ return 0
++++ '[' LoadBalancer = ClusterIP ']'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}'
++++ egrep -q 'hostname|ip'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.3l28bjuQD1
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.WEJ4GQ2bDI
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.3l28bjuQD1
++++ cat /tmp/tmp.WEJ4GQ2bDI
++++ rm /tmp/tmp.3l28bjuQD1 /tmp/tmp.WEJ4GQ2bDI
++++ return 0
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.s1gUmdWCEk
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.mHVbqqb8Rk
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.s1gUmdWCEk
++++ cat /tmp/tmp.mHVbqqb8Rk
++++ rm /tmp/tmp.s1gUmdWCEk /tmp/tmp.mHVbqqb8Rk
++++ return 0
+++ kubectl_bin exec -n monitoring-2-0-932 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.61.206.88/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.OSQ2QrOZPZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IoVW6CFYGh
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-932 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.61.206.88/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.OSQ2QrOZPZ
+++ cat /tmp/tmp.IoVW6CFYGh
+++ rm /tmp/tmp.OSQ2QrOZPZ /tmp/tmp.IoVW6CFYGh
+++ return 0
++ echo /node_id/0c18f111-be34-4729-8701-2d53bd7f9341 /node_id/a4ffb25e-4a64-4317-acf9-6f0ae377e62a /node_id/7ac820ba-94f1-4a9f-9d8d-3e1a07ca929d
+ for node_id in '"${nodeList_from_pmm[@]}"'
+ '[' -z /node_id/0c18f111-be34-4729-8701-2d53bd7f9341 ']'
+ for node_id in '"${nodeList_from_pmm[@]}"'
+ '[' -z /node_id/a4ffb25e-4a64-4317-acf9-6f0ae377e62a ']'
+ for node_id in '"${nodeList_from_pmm[@]}"'
+ '[' -z /node_id/7ac820ba-94f1-4a9f-9d8d-3e1a07ca929d ']'
+ kubectl_bin patch pxc monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.s6Mq93ctdf
++ mktemp
+ local LAST_ERR=/tmp/tmp.77SgRiWOTR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc monitoring --type json '-p=[{"op":"add","path":"/spec/pause","value":true}]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.s6Mq93ctdf
perconaxtradbcluster.pxc.percona.com/monitoring patched
+ cat /tmp/tmp.77SgRiWOTR
+ rm /tmp/tmp.s6Mq93ctdf /tmp/tmp.77SgRiWOTR
+ return 0
+ wait_for_delete pod/monitoring-pxc-0
+ local res=pod/monitoring-pxc-0
+ echo -n 'waiting for pod/monitoring-pxc-0 to be deleted'
waiting for pod/monitoring-pxc-0 to be deleted+ set +o xtrace
................Error from server (NotFound): pods "monitoring-pxc-0" not found
+ does_node_id_exists_in_pmm=($(does_node_id_exists "${nodeList[@]}"))
++ does_node_id_exists /node_id/0c18f111-be34-4729-8701-2d53bd7f9341 /node_id/a4ffb25e-4a64-4317-acf9-6f0ae377e62a /node_id/7ac820ba-94f1-4a9f-9d8d-3e1a07ca929d
++ nodeList=("$@")
++ local -a nodeList
++ nodeList_from_pmm=()
++ local -a nodeList_from_pmm
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ grep -q NotFound
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
+++ grep /node_id/0c18f111-be34-4729-8701-2d53bd7f9341
+++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.bi2IZ7HISh
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.Hqrw6H8gRm
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.bi2IZ7HISh
+++++ cat /tmp/tmp.Hqrw6H8gRm
+++++ rm /tmp/tmp.bi2IZ7HISh /tmp/tmp.Hqrw6H8gRm
+++++ return 0
++++ '[' LoadBalancer = ClusterIP ']'
++++ egrep -q 'hostname|ip'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.aTk6cRR6DO
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.a02SynOlzt
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.aTk6cRR6DO
++++ cat /tmp/tmp.a02SynOlzt
++++ rm /tmp/tmp.aTk6cRR6DO /tmp/tmp.a02SynOlzt
++++ return 0
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.pfkp4HZYzh
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.6YvdOLDgYZ
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.pfkp4HZYzh
++++ cat /tmp/tmp.6YvdOLDgYZ
++++ rm /tmp/tmp.pfkp4HZYzh /tmp/tmp.6YvdOLDgYZ
++++ return 0
+++ kubectl_bin exec -n monitoring-2-0-932 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.61.206.88/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.m1yLWpGIyJ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.L4VxdjJmRp
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-932 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.61.206.88/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.m1yLWpGIyJ
+++ cat /tmp/tmp.L4VxdjJmRp
+++ rm /tmp/tmp.m1yLWpGIyJ /tmp/tmp.L4VxdjJmRp
+++ return 0
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/a4ffb25e-4a64-4317-acf9-6f0ae377e62a
+++ awk '{print $4}'
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ grep -q NotFound
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
+++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.FyXeVfvoUb
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.JmkYj6MWjs
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.FyXeVfvoUb
+++++ cat /tmp/tmp.JmkYj6MWjs
+++++ rm /tmp/tmp.FyXeVfvoUb /tmp/tmp.JmkYj6MWjs
+++++ return 0
++++ '[' LoadBalancer = ClusterIP ']'
++++ egrep -q 'hostname|ip'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.aRq1XQjiqT
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.wQGfD3MDJK
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.aRq1XQjiqT
++++ cat /tmp/tmp.wQGfD3MDJK
++++ rm /tmp/tmp.aRq1XQjiqT /tmp/tmp.wQGfD3MDJK
++++ return 0
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.f6azw6sHNn
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.QAJpElwYu5
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.f6azw6sHNn
++++ cat /tmp/tmp.QAJpElwYu5
++++ rm /tmp/tmp.f6azw6sHNn /tmp/tmp.QAJpElwYu5
++++ return 0
+++ kubectl_bin exec -n monitoring-2-0-932 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.61.206.88/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6CQIi9Bgqu
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NtHg970gPF
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-932 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.61.206.88/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.6CQIi9Bgqu
+++ cat /tmp/tmp.NtHg970gPF
+++ rm /tmp/tmp.6CQIi9Bgqu /tmp/tmp.NtHg970gPF
+++ return 0
++ for node_id in '"${nodeList[@]}"'
++ nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
+++ grep /node_id/7ac820ba-94f1-4a9f-9d8d-3e1a07ca929d
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
+++ awk '{print $4}'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
++++ grep -q NotFound
+++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.yygPmJ2NpR
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.HXnQZeP4TK
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.yygPmJ2NpR
+++++ cat /tmp/tmp.HXnQZeP4TK
+++++ rm /tmp/tmp.yygPmJ2NpR /tmp/tmp.HXnQZeP4TK
+++++ return 0
++++ '[' LoadBalancer = ClusterIP ']'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}'
++++ egrep -q 'hostname|ip'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.S9S6XXWT1g
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.XOLYEOEL5R
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.S9S6XXWT1g
++++ cat /tmp/tmp.XOLYEOEL5R
++++ rm /tmp/tmp.S9S6XXWT1g /tmp/tmp.XOLYEOEL5R
++++ return 0
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.hqvJgSZNC9
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.zsUjX1QoKW
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.hqvJgSZNC9
++++ cat /tmp/tmp.zsUjX1QoKW
++++ rm /tmp/tmp.hqvJgSZNC9 /tmp/tmp.zsUjX1QoKW
++++ return 0
+++ kubectl_bin exec -n monitoring-2-0-932 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.61.206.88/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.i2CbazVll3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.aaygZxkL6x
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec -n monitoring-2-0-932 monitoring-0 -- pmm-admin --server-url=https://admin:admin@34.61.206.88/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.i2CbazVll3
+++ cat /tmp/tmp.aaygZxkL6x
+++ rm /tmp/tmp.i2CbazVll3 /tmp/tmp.aaygZxkL6x
+++ return 0
++ echo
+ [[ -n '' ]]
+ helm uninstall monitoring
release "monitoring" uninstalled
+ destroy monitoring-2-0-932
+ local namespace=monitoring-2-0-932
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ tee /tmp/tmp.h65oP78O1Z/operator.log
+ grep -v 'get backup status: Job.batch'
+ grep -v 'the object has been modified'
+ grep -v level=info
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ sort -u
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IrHoqP4n30
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3Gjw2XTNIm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IrHoqP4n30
++ cat /tmp/tmp.3Gjw2XTNIm
++ rm /tmp/tmp.IrHoqP4n30 /tmp/tmp.3Gjw2XTNIm
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-f5b849cf6-jlmmq
++ mktemp
+ local LAST_OUT=/tmp/tmp.vYSpiZuwhk
++ mktemp
+ local LAST_ERR=/tmp/tmp.d9wbknvRXE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-f5b849cf6-jlmmq
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vYSpiZuwhk
+ cat /tmp/tmp.d9wbknvRXE
+ rm /tmp/tmp.vYSpiZuwhk /tmp/tmp.d9wbknvRXE
+ return 0

  }
  	},
  		},
  			{
  			},
  				},
  					{
  					},
  						},
  							{
  							},
  								},
  									},
- 		},
- 			{
- 			},
+ 		},
  							... // 11 identical elements
  							... // 12 identical elements
  						... // 16 identical fields
  							... // 16 identical fields
2025-10-15T10:24:24.524Z	INFO	setup	Manager starting up	{"gitCommit": "6c08ea71a9c64b418e03f427c80eeb7b401eee22", "gitBranch": "PR-2213-6c08ea71", "buildTime": "2025-10-15T09:27:18Z", "goVersion": "go1.24.9", "os": "linux", "arch": "amd64"}
2025-10-15T10:24:24.524Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.31.13-gke.1023000"}
2025-10-15T10:24:24.527Z	INFO	setup	Registering Components.
2025-10-15T10:24:28.589Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2025-10-15T10:24:28.589Z	INFO	controller-runtime.metrics	Starting metrics server
2025-10-15T10:24:28.589Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2025-10-15T10:24:28.589Z	INFO	setup	Starting the Cmd.
2025-10-15T10:24:28.589Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2025-10-15T10:24:28.590Z	INFO	controller-runtime.certwatcher	Starting certificate poll+watcher	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"}
2025-10-15T10:24:28.590Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"}
2025-10-15T10:24:28.590Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2025-10-15T10:24:28.590Z	INFO	controller-runtime.webhook	Starting webhook server
2025-10-15T10:24:28.691Z	INFO	attempting to acquire leader lease pxc-operator/08db1feb.percona.com...
2025-10-15T10:24:28.727Z	DEBUG	events	percona-xtradb-cluster-operator-f5b849cf6-jlmmq_343a420c-07f2-4195-b63a-baeb4a976c61 became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"9bc28220-0f43-4c2a-b6f7-71feb0d9646a","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1760523868722511009"}, "reason": "LeaderElection"}
2025-10-15T10:24:28.727Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2025-10-15T10:24:28.727Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2025-10-15T10:24:28.727Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.Secret"}
2025-10-15T10:24:28.727Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2025-10-15T10:24:28.727Z	INFO	successfully acquired lease pxc-operator/08db1feb.percona.com
2025-10-15T10:24:28.828Z	INFO	Starting Controller	{"controller": "pxcbackup-controller"}
2025-10-15T10:24:28.828Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2025-10-15T10:24:28.828Z	INFO	Starting Controller	{"controller": "pxcrestore-controller"}
2025-10-15T10:24:28.828Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "worker count": 1}
2025-10-15T10:24:28.828Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2025-10-15T10:24:28.828Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "worker count": 1}
2025-10-15T10:25:42.080Z	INFO	Set CR version	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "73f74232-416b-4147-860d-db7e30a8cbdf", "version": "1.19.0"}
2025-10-15T10:25:45.623Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "73f74232-416b-4147-860d-db7e30a8cbdf", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2025-10-15T10:25:45.661Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "73f74232-416b-4147-860d-db7e30a8cbdf", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2025-10-15T10:25:45.755Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "73f74232-416b-4147-860d-db7e30a8cbdf", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2025-10-15T10:25:45.799Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "73f74232-416b-4147-860d-db7e30a8cbdf", "object": "monitoring-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2025-10-15T10:25:45.880Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "73f74232-416b-4147-860d-db7e30a8cbdf", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2025-10-15T10:25:45.973Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "73f74232-416b-4147-860d-db7e30a8cbdf", "object": "monitoring-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2025-10-15T10:25:47.500Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "38515349-7694-453e-8285-3f7289b4228b", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2025-10-15T10:25:47.563Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "38515349-7694-453e-8285-3f7289b4228b", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2025-10-15T10:27:03.423Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d93711c8-caaf-48f2-8f03-9ccc4406c6b1", "user": "operator"}
2025-10-15T10:27:03.468Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d93711c8-caaf-48f2-8f03-9ccc4406c6b1", "user": "monitor"}
2025-10-15T10:27:03.513Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d93711c8-caaf-48f2-8f03-9ccc4406c6b1"}
2025-10-15T10:27:03.549Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d93711c8-caaf-48f2-8f03-9ccc4406c6b1"}
2025-10-15T10:27:03.578Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d93711c8-caaf-48f2-8f03-9ccc4406c6b1", "user": "xtrabackup"}
2025-10-15T10:27:03.630Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d93711c8-caaf-48f2-8f03-9ccc4406c6b1"}
2025-10-15T10:27:03.665Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d93711c8-caaf-48f2-8f03-9ccc4406c6b1", "user": "replication"}
2025-10-15T10:29:34.446Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "97f046b0-bb25-47e2-b18c-c1d031376f61", "user": "root"}
2025-10-15T10:29:34.520Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "97f046b0-bb25-47e2-b18c-c1d031376f61", "new version": "8.0.43-34.1"}
2025-10-15T10:32:32.305Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "08f62b9b-94b0-4390-ad14-57295976bf91", "user": "pmmserverkey"}
2025-10-15T10:32:34.402Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "52bd2c51-a308-4bf5-9cbf-fdd51c4be30b", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-10-15T10:32:34.487Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "52bd2c51-a308-4bf5-9cbf-fdd51c4be30b", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-10-15T10:32:34.554Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "52bd2c51-a308-4bf5-9cbf-fdd51c4be30b", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-10-15T10:33:33.227Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "5cd0f216-4447-4905-a5aa-f5695ece8699", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-1: dial tcp: lookup monitoring-pxc-1.monitoring-pxc.monitoring-2-0-932 on 34.118.224.10:53: no such host"}
2025-10-15T10:34:41.018Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "2080eed0-8af9-431f-b004-aa8464d1d0af", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-0: dial tcp: lookup monitoring-pxc-0.monitoring-pxc.monitoring-2-0-932 on 34.118.224.10:53: no such host"}
2025-10-15T10:35:18.375Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "fccb3aaa-f645-40e8-831a-232bd1ba3e30", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-10-15T10:35:18.460Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "fccb3aaa-f645-40e8-831a-232bd1ba3e30", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-10-15T10:35:18.645Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "fccb3aaa-f645-40e8-831a-232bd1ba3e30", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-10-15T10:36:16.369Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "0d47c317-27ed-479f-b1f5-99a492a7c361", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-1: dial tcp: lookup monitoring-pxc-1.monitoring-pxc.monitoring-2-0-932 on 34.118.224.10:53: no such host"}
2025-10-15T10:37:48.148Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d4bd6422-e4b4-4ed0-b81a-739a6a43750f", "user": "pmmserverkey"}
2025-10-15T10:37:48.179Z	INFO	HAProxy pods will be restarted	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d4bd6422-e4b4-4ed0-b81a-739a6a43750f", "last-applied-secret": "87201833e9a106c1b4a258b24a40c108cdcc79d39702bb431c336fd1744a1359"}
2025-10-15T10:37:48.179Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d4bd6422-e4b4-4ed0-b81a-739a6a43750f", "user": "pmmserverkey"}
2025-10-15T10:37:48.179Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d4bd6422-e4b4-4ed0-b81a-739a6a43750f", "last-applied-secret": "87201833e9a106c1b4a258b24a40c108cdcc79d39702bb431c336fd1744a1359"}
2025-10-15T10:37:48.181Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d4bd6422-e4b4-4ed0-b81a-739a6a43750f", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-10-15T10:37:48.252Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "d4bd6422-e4b4-4ed0-b81a-739a6a43750f", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-10-15T10:38:50.564Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "87935505-4bd9-4ca3-b09e-44d61bfd88bc", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-1: dial tcp: lookup monitoring-pxc-1.monitoring-pxc.monitoring-2-0-932 on 34.118.224.10:53: no such host"}
2025-10-15T10:39:53.861Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "da7945b0-6631-44d1-ada7-fb69e5ad11e1", "err": "failed to ensure cluster readonly status: connect to pod monitoring-pxc-0: dial tcp: lookup monitoring-pxc-0.monitoring-pxc.monitoring-2-0-932 on 34.118.224.10:53: no such host"}
2025-10-15T10:44:40.785Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "4e7b7540-e3ee-44da-b99b-1a8597441942", "object": "monitoring-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-10-15T10:44:40.856Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "4e7b7540-e3ee-44da-b99b-1a8597441942", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
2025-10-15T10:44:40.920Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "monitoring-2-0-932", "name": "monitoring", "reconcileID": "4e7b7540-e3ee-44da-b99b-1a8597441942", "object": "monitoring-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true}
  							... // 22 identical elements
  							... // 22 identical fields
  							... // 23 identical elements
  							... // 25 identical elements
  							... // 26 identical elements
  					... // 2 identical entries
  		... // 2 identical fields
  				... // 2 identical fields
  						... // 2 identical fields
  									... // 2 identical fields
  					... // 3 identical fields
  						... // 3 identical fields
  							... // 3 identical fields
  							... // 4 identical elements
  						... // 4 identical fields
  							... // 4 identical fields
  				... // 5 identical fields
  					... // 5 identical fields
  				... // 6 identical fields
  				... // 7 identical fields
  						... // 8 identical fields
  							... // 9 identical elements
  				... // 9 identical fields
  							... // 9 identical fields
  					AccessModes: nil,
  				ActiveDeadlineSeconds:         nil,
  				Affinity:              nil,
  				Affinity:      nil,
  				Annotations: map[string]string{
- 		Annotations: map[string]string{
+ 		Annotations: map[string]string{
+ 										APIVersion: "",
- 				APIVersion:  "apps/v1",
- 				APIVersion: "apps/v1",
- 										APIVersion: "v1",
  						Args:       {"haproxy"},
  						Args:       {"mysqld"},
  						Args:       nil,
  				AutomountServiceAccountToken:  nil,
+ 		AvailableReplicas:  0,
- 		AvailableReplicas:  2,
- 		AvailableReplicas:  3,
  							AWSElasticBlockStore: nil,
  							AzureFile: nil,
  					Capacity:    nil,
- 		CollisionCount:     &0,
+ 		CollisionCount:     nil,
  		Conditions:         nil,
  									ConfigMapKeyRef:  nil,
  							ConfigMap: &v1.ConfigMapVolumeSource{
  								ContainerPort: 30100,
  								ContainerPort: 30101,
  								ContainerPort: 30102,
  								ContainerPort: 30103,
  								ContainerPort: 30104,
  								ContainerPort: 30105,
  								ContainerPort: 3306,
  								ContainerPort: 33060,
  								ContainerPort: 33062,
  								ContainerPort: 3307,
  								ContainerPort: 3309,
  								ContainerPort: 4444,
  								ContainerPort: 4567,
  								ContainerPort: 4568,
  								ContainerPort: 7777,
  								ContainerPort: 8404,
  				Containers: []v1.Container{
+ 		CreationTimestamp:          v1.Time{},
- 		CreationTimestamp:          v1.Time{Time: s"2025-10-15 10:25:45 +0000 UTC"},
+ 		CurrentReplicas:    0,
- 		CurrentReplicas:    2,
- 		CurrentReplicas:    3,
+ 		CurrentRevision:    "",
- 		CurrentRevision:    "monitoring-haproxy-5fc9ddf657",
- 		CurrentRevision:    "monitoring-haproxy-7c56cc486c",
- 		CurrentRevision:    "monitoring-haproxy-7d88c96f4c",
- 		CurrentRevision:    "monitoring-haproxy-865847fd75",
- 		CurrentRevision:    "monitoring-pxc-65449d488",
- 		CurrentRevision:    "monitoring-pxc-65896bc69d",
- 		CurrentRevision:    "monitoring-pxc-6899cbbd7b",
- 		CurrentRevision:    "monitoring-pxc-866c4cc6fb",
  					DataSource:                nil,
  					DataSourceRef:             nil,
- 								DefaultMode:          &420,
- 								DefaultMode: &420,
+ 								DefaultMode:          nil,
+ 								DefaultMode: nil,
  				DeletionGracePeriodSeconds: nil,
  		DeletionGracePeriodSeconds: nil,
  		DeletionTimestamp:          nil,
+ 				DeprecatedServiceAccount:      "",
- 				DeprecatedServiceAccount:      "default",
+ 				DNSPolicy:                     "",
- 				DNSPolicy:                     "ClusterFirst",
  						EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "my-env-var-secrets"}, Optional: &true}}},
  						Env:     {{Name: "PXC_SERVICE", Value: "monitoring-pxc"}, {Name: "LIVENESS_CHECK_TIMEOUT", Value: "5"}, {Name: "READINESS_CHECK_TIMEOUT", Value: "1"}},
  						Env:     {{Name: "PXC_SERVICE", Value: "monitoring-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-monitoring"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-monitoring"}, Key: "xtrabackup"}}}, ...},
  						Env: []v1.EnvVar{
  				EphemeralContainers:           nil,
  								Exec: nil,
  							FailureThreshold:              3,
  							FC:        nil,
  										FieldPath:  "metadata.name",
  										FieldPath:  "metadata.namespace",
  									FieldRef:         nil,
  									FieldRef: &v1.ObjectFieldSelector{
- 				FieldsType:  "FieldsV1",
- 				FieldsType: "FieldsV1",
- 				FieldsV1:   s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`...,
- 				FieldsV1:    s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`...,
  									FileKeyRef: nil,
  				Finalizers:      nil,
  		Finalizers:      nil,
+ 		Generation:                 0,
- 		Generation:                 1,
- 		Generation:                 2,
- 		Generation:                 3,
- 		Generation:                 4,
  							GitRepo:              nil,
  								GRPC:      nil,
  									Host:        "",
  				HostAliases:           nil,
  				HostAliases:   nil,
  								HostIP:        "",
  				HostIPC:               false,
  				Hostname:              "",
  								HostPort:      0,
  								HTTPGet: &v1.HTTPGetAction{
  									HTTPHeaders: nil,
  						ImagePullPolicy:          "Always",
  				ImagePullSecrets:      nil,
  				InitContainers: []v1.Container{
  							InitialDelaySeconds: 15,
  							InitialDelaySeconds:           300,
  							InitialDelaySeconds: 300,
  							ISCSI: nil,
  								Items:                nil,
  								Items:       nil,
- 										Key:                  "pmmserver",
+ 										Key:                  "pmmserverkey",
  					"kubectl.kubernetes.io/default-container": "haproxy",
  					"kubectl.kubernetes.io/default-container": "pxc",
  				Labels:                     {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...},
  				Labels:                     {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...},
  		Labels:                     nil,
+ 					"last-applied-secret":                     "87201833e9a106c1b4a258b24a40c108cdcc79d39702bb431c336fd1744a1359",
  						Lifecycle:                nil,
  						Lifecycle:                &{PreStop: &{Exec: &{Command: {"bash", "-c", "pmm-admin unregister --force"}}}},
  						LivenessProbe: &v1.Probe{
  								LocalObjectReference: {Name: "auto-monitoring-pxc"},
  										LocalObjectReference: {Name: "internal-monitoring"},
  								LocalObjectReference: {Name: "monitoring-haproxy"},
  								LocalObjectReference: {Name: "monitoring-pxc"},
  				ManagedFields:   nil,
+ 		ManagedFields: nil,
- 		ManagedFields: []v1.ManagedFieldsEntry{
- 				Manager:     "kube-controller-manager",
- 				Manager:    "percona-xtradb-cluster-operator",
  		MinReadySeconds:                      0,
[mysql] 2025/10/15 10:34:39 packets.go:58 read tcp 10.139.200.30:60712->34.118.226.131:3306: i/o timeout
[mysql] 2025/10/15 10:37:20 packets.go:58 unexpected EOF
[mysql] 2025/10/15 10:39:38 packets.go:58 unexpected EOF
[mysql] 2025/10/15 10:39:52 packets.go:58 unexpected EOF
  								Name:          "",
  						Name: "auto-config",
  					{Name: "bin", VolumeSource: {EmptyDir: &{}}},
  							{Name: "CLIENT_PORT_LISTEN", Value: "7777"},
  							{Name: "CLIENT_PORT_MAX", Value: "30105"},
  							{Name: "CLIENT_PORT_MIN", Value: "30100"},
  						Name: "config",
  							{Name: "DB_TYPE", Value: "haproxy"},
  							{Name: "DB_TYPE", Value: "mysql"},
  							{Name: "DB_USER", Value: "monitor"},
  					{Name: "haproxy-auto", VolumeSource: {EmptyDir: &{}}},
  						Name: "haproxy-custom",
  								Name:          "ist",
  							{Name: "MONITOR_USER", Value: "monitor"},
  						Name: "my-env-var-secrets",
  								Name:          "mysql",
  								Name:          "mysql-admin",
  						Name: "mysql-init-file",
  								Name:          "mysql-replicas",
  						Name: "mysql-users-secret-file",
  								Name:          "mysqlx",
  							{Name: "PMM_AGENT_LISTEN_PORT", Value: "7777"},
  							{Name: "PMM_AGENT_PORTS_MIN", Value: "30100"},
  							{Name: "PMM_AGENT_SERVER_ADDRESS", Value: "monitoring-service"},
  								Name:  "PMM_AGENT_SERVER_PASSWORD",
  								Name:      "PMM_AGENT_SERVER_USERNAME",
  							{Name: "PMM_AGENT_SERVER_USERNAME", Value: "api_key"},
  							{Name: "PMM_AGENT_SETUP_FORCE", Value: "1"},
  								Name:      "PMM_AGENT_SETUP_NODE_NAME",
  							{Name: "PMM_AGENT_SETUP_NODE_TYPE", Value: "container"},
  								Name:  "PMM_PASSWORD",
  							{Name: "PMM_SERVER", Value: "monitoring-service"},
  								Name:      "PMM_USER",
  								Name:  "POD_NAME",
  								Name:  "POD_NAMESPASE",
  								Name:          "proxy-protocol",
  		Namespace:                  "monitoring-2-0-932",
  						Name: "ssl",
  						Name: "ssl-internal",
  								Name:          "sst",
  								Name:          "stats",
  					{Name: "tmp", VolumeSource: {EmptyDir: &{}}},
  						Name: "vault-keyring-secret",
  								Name:          "write-set",
  							NFS:   nil,
  				NodeName:                      "",
  				NodeSelector:                  nil,
  			ObjectMeta: {Labels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "haproxy", "last-applied-secret": "87201833e9a106c1b4a258b24a40c108cdcc79d39702bb431c336fd1744a1359", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0"}},
  			ObjectMeta: {Labels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "haproxy", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e"}},
  			ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "last-applied-secret": "87201833e9a106c1b4a258b24a40c108cdcc79d39702bb431c336fd1744a1359", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/env-secret-config-hash": "5bc403a8d44324c1bc704f02e74876d0", ...}},
  			ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "f186582d3f04a99de33e3a9e33e481c6", "percona.com/ssl-internal-hash": "a97287191b1a8f922ca3c4de7f3e7a19"}},
  				ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},
  			ObjectMeta: v1.ObjectMeta{
  	ObjectMeta: v1.ObjectMeta{
+ 		ObservedGeneration: 0,
- 		ObservedGeneration: 1,
- 		ObservedGeneration: 2,
- 		ObservedGeneration: 3,
- 		ObservedGeneration: 4,
- 				Operation:   "Update",
- 				Operation:  "Update",
  								Optional:    &false,
  										Optional:             nil,
  								Optional:             &true,
  								Optional:    &true,
  		Ordinals:                             nil,
  				OS:                        nil,
  				Overhead:                  nil,
  		OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "monitoring", UID: "21753d79-60bf-48eb-b270-e206173a0a30", ...}},
  				OwnerReferences: nil,
  									Path:        "/local/Status",
  					"percona.com/configuration-hash":          "d41d8cd98f00b204e9800998ecf8427e",
  					"percona.com/env-secret-config-hash":      "5bc403a8d44324c1bc704f02e74876d0",
+ 					"percona.com/env-secret-config-hash":      "5bc403a8d44324c1bc704f02e74876d0",
+ 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBw"...,
+ 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3Vi"...,
- 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBw"...,
+ 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsImxhc3QtYXBwbGllZC1zZWNyZXQiOiI4NzIwMTgzM2U5YTEwNmMxYjRhMjU4YjI0YTQwYzEwOGNkY2M3OWQzOTcwMmJiNDMxYzMzNmZkMTc0NGExMzU5IiwicGVyY29uYS5jb20vY29uZmlndXJhdGlvbi1oYXNoIjoi"...,
- 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFi"...,
+ 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFi"...,
- 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJoYXByb3h5LWN1c3RvbSIsImNvbmZpZ01h"...,
- 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJoYXByb3h5LWN1c3RvbSIsImNvbmZpZ01hcCI6eyJuYW1lIjoibW9uaXRvcmluZy1oYXByb3h5Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJoYXByb3h5LWF1dG8iLCJlbXB0eURpciI6e319LHsibmFtZSI6Im15c3FsLXVzZXJzLXNlY3JldC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXktZW52LXZhci1zZWNyZXRzIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJteS1lbnYtdmFyLXNlY3JldHMiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6ImJpbiIsImVtcHR5RGlyIjp7fX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjIxMy02YzA4ZWE3MSIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifSx7Im5hbWUiOiJoYXByb3h5LWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yMjEzLTZjMDhlYTcxIiwiY29tbWFuZCI6WyIvaGFwcm94eS1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii9vcHQvcGVyY29uYSJ9XSwiaW1hZ2VQdWxsUG9saWN5IjoiQWx3YXlzIn1dLCJjb250YWluZXJzIjpbeyJuYW1lIjoicG1tLWNsaWVudCIsImltYWdlIjoicGVyY29uYWxhYi9wbW0tY2xpZW50OmRldi1sYXRlc3QiLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6Nzc3N30seyJjb250YWluZXJQb3J0IjozMDEwMH0seyJjb250YWluZXJQb3J0IjozMDEwMX0seyJjb250YWluZXJQb3J0IjozMDEwMn0seyJjb250YWluZXJQb3J0IjozMDEwM30seyJjb250YWluZXJQb3J0IjozMDEwNH0seyJjb250YWluZXJQb3J0IjozMDEwNX1dLCJlbnZGcm9tIjpbeyJzZWNyZXRSZWYiOnsibmFtZSI6Im15LWVudi12YXItc2VjcmV0cyIsIm9wdGlvbmFsIjp0cnVlfX1dLCJlbnYiOlt7Im5hbWUiOiJQTU1fU0VSVkVSIiwidmFsdWUiOiJtb25pdG9yaW5nLXNlcnZpY2UifSx7Im5hbWUiOiJQTU1fVVNFUiIsInZhbHVlIjoiYWRtaW4ifSx7Im5hbWUiOiJQTU1fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwia2V5IjoicG1tc2VydmVyIn19fSx7Im5hbWUiOiJDTElFTlRfUE9SVF9MSVNURU4iLCJ2YWx1ZSI6Ijc3NzcifSx7Im5hbWUiOiJDTElFTlRfUE9SVF9NSU4iLCJ2YWx1ZSI6IjMwMTAwIn0seyJuYW1lIjoiQ0xJRU5UX1BPUlRfTUFYIiwidmFsdWUiOiIzMDEwNSJ9LHsibmFtZSI6IlBPRF9OQU1FIiwidmFsdWVGcm9tIjp7ImZpZWxkUmVmIjp7ImZpZWxkUGF0aCI6Im1ldGFkYXRhLm5hbWUifX19LHsibmFtZSI6IlBPRF9OQU1FU1BBU0UiLCJ2YWx1ZUZyb20iOnsiZmllbGRSZWYiOnsiZmllbGRQYXRoIjoibWV0YWRhdGEubmFtZXNwYWNlIn19fSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX0FERFJFU1MiLCJ2YWx1ZSI6Im1vbml0b3Jpbmctc2VydmljZSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfVVNFUk5BTUUiLCJ2YWx1ZSI6ImFkbWluIn0seyJuYW1lIjoiUE1NX0FHRU5UX1NFUlZFUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLW1vbml0b3JpbmciLCJrZXkiOiJwbW1zZXJ2ZXIifX19LHsibmFtZSI6IlBNTV9BR0VOVF9MSVNURU5fUE9SVCIsInZhbHVlIjoiNzc3NyJ9LHsibmFtZSI6IlBNTV9BR0VOVF9QT1JUU19NSU4iLCJ2YWx1ZSI6IjMwMTAwIn0seyJuYW1lIjoiUE1NX0FHRU5UX1BPUlRTX01BWCIsInZhbHVlIjoiMzAxMDUifSx7Im5hbWUiOiJQTU1fQUdFTlRfQ09ORklHX0ZJTEUiLCJ2YWx1ZSI6Ii91c3IvbG9jYWwvcGVyY29uYS9wbW0yL2NvbmZpZy9wbW0tYWdlbnQueWFtbCJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfSU5TRUNVUkVfVExTIiwidmFsdWUiOiIxIn0seyJuYW1lIjoiUE1NX0FHRU5UX0xJU1RFTl9BRERSRVNTIiwidmFsdWUiOiIwLjAuMC4wIn0seyJuYW1lIjoiUE1NX0FHRU5UX1NFVFVQX01FVFJJQ1NfTU9ERSIsInZhbHVlIjoicHVz"...,
+ 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Miwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoiaGFwcm94eSIsImFwcC5rdWJlcm5ldGVzLmlvL2luc3RhbmNlIjoibW9uaXRvcmluZyIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6ImhhcHJveHkiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6Im1vbml0b3JpbmciLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoiaGFwcm94eSIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJoYXByb3h5LWN1c3RvbSIsImNvbmZpZ01hcCI6eyJuYW1lIjoibW9uaXRvcmluZy1oYXByb3h5Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJoYXByb3h5LWF1dG8iLCJlbXB0eURpciI6e319LHsibmFtZSI6Im15c3FsLXVzZXJzLXNlY3JldC1maWxlIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoibXktZW52LXZhci1zZWNyZXRzIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJteS1lbnYtdmFyLXNlY3JldHMiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6ImJpbiIsImVtcHR5RGlyIjp7fX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjIxMy02YzA4ZWE3MSIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifSx7Im5hbWUiOiJoYXByb3h5LWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yMjEzLTZjMDhlYTcxIiwiY29tbWFuZCI6WyIvaGFwcm94eS1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii9vcHQvcGVyY29uYSJ9XSwiaW1hZ2VQdWxsUG9saWN5IjoiQWx3YXlzIn1dLCJjb250YWluZXJzIjpbeyJuYW1lIjoicG1tLWNsaWVudCIsImltYWdlIjoicGVyY29uYWxhYi9wbW0tY2xpZW50OmRldi1sYXRlc3QiLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6Nzc3N30seyJjb250YWluZXJQb3J0IjozMDEwMH0seyJjb250YWluZXJQb3J0IjozMDEwMX0seyJjb250YWluZXJQb3J0IjozMDEwMn0seyJjb250YWluZXJQb3J0IjozMDEwM30seyJjb250YWluZXJQb3J0IjozMDEwNH0seyJjb250YWluZXJQb3J0IjozMDEwNX1dLCJlbnZGcm9tIjpbeyJzZWNyZXRSZWYiOnsibmFtZSI6Im15LWVudi12YXItc2VjcmV0cyIsIm9wdGlvbmFsIjp0cnVlfX1dLCJlbnYiOlt7Im5hbWUiOiJQTU1fU0VSVkVSIiwidmFsdWUiOiJtb25pdG9yaW5nLXNlcnZpY2UifSx7Im5hbWUiOiJQTU1fVVNFUiIsInZhbHVlIjoiYXBpX2tleSJ9LHsibmFtZSI6IlBNTV9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLW1vbml0b3JpbmciLCJrZXkiOiJwbW1zZXJ2ZXJrZXkifX19LHsibmFtZSI6IkNMSUVOVF9QT1JUX0xJU1RFTiIsInZhbHVlIjoiNzc3NyJ9LHsibmFtZSI6IkNMSUVOVF9QT1JUX01JTiIsInZhbHVlIjoiMzAxMDAifSx7Im5hbWUiOiJDTElFTlRfUE9SVF9NQVgiLCJ2YWx1ZSI6IjMwMTA1In0seyJuYW1lIjoiUE9EX05BTUUiLCJ2YWx1ZUZyb20iOnsiZmllbGRSZWYiOnsiZmllbGRQYXRoIjoibWV0YWRhdGEubmFtZSJ9fX0seyJuYW1lIjoiUE9EX05BTUVTUEFTRSIsInZhbHVlRnJvbSI6eyJmaWVsZFJlZiI6eyJmaWVsZFBhdGgiOiJtZXRhZGF0YS5uYW1lc3BhY2UifX19LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfQUREUkVTUyIsInZhbHVlIjoibW9uaXRvcmluZy1zZXJ2aWNlIn0seyJuYW1lIjoiUE1NX0FHRU5UX1NFUlZFUl9VU0VSTkFNRSIsInZhbHVlIjoiYXBpX2tleSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1tb25pdG9yaW5nIiwia2V5IjoicG1tc2VydmVya2V5In19fSx7Im5hbWUiOiJQTU1fQUdFTlRfTElTVEVOX1BPUlQiLCJ2YWx1ZSI6Ijc3NzcifSx7Im5hbWUiOiJQTU1fQUdFTlRfUE9SVFNfTUlOIiwidmFsdWUiOiIzMDEwMCJ9LHsibmFtZSI6IlBNTV9BR0VOVF9QT1JUU19NQVgiLCJ2YWx1ZSI6IjMwMTA1In0seyJuYW1lIjoiUE1NX0FHRU5UX0NPTkZJR19GSUxFIiwidmFsdWUiOiIvdXNyL2xvY2FsL3BlcmNvbmEvcG1tMi9jb25maWcvcG1tLWFnZW50LnlhbWwifSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX0lOU0VDVVJFX1RMUyIsInZhbHVlIjoiMSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9MSVNURU5fQUREUkVTUyIsInZhbHVlIjoiMC4wLjAuMCJ9LHsibmFtZSI6IlBNTV9BR0VOVF9TRVRVUF9NRVRSSUNTX01PREUiLCJ2"...,
- 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3Vi"...,
+ 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsImxhc3QtYXBwbGllZC1zZWNyZXQiOiI4NzIwMTgzM2U5YTEwNmMxYjRhMjU4YjI0YTQwYzEwOGNkY2M3OWQzOTcwMmJiNDMxYzMzNmZkMTc0NGExMzU5IiwicGVyY29uYS5jb20vY29uZmlndXJhdGlvbi1oYXNoIjoiZDQxZDhjZDk4ZjAw"...,
- 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vc3NsLWhhc2giOiJmMTg2NTgyZDNmMDRhOTlkZTMzZTNhOWUzM2U0ODFjNiIsInBlcmNvbmEu"...,
+ 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vc3NsLWhhc2giOiJmMTg2NTgyZDNmMDRhOTlkZTMzZTNhOWUzM2U0ODFjNiIsInBlcmNvbmEuY29tL3NzbC1pbnRlcm5hbC1oYXNoIjoiYTk3Mjg3MTkxYjFhOGY5MjJjYTNjNGRlN2YzZTdhMTkifX0sInNwZWMiOnsidm9sdW1lcyI6W3sibmFtZSI6InRtcCIsImVtcHR5RGlyIjp7fX0seyJuYW1lIjoiY29uZmlnIiwiY29uZmlnTWFwIjp7Im5hbWUiOiJtb25pdG9yaW5nLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJtb25pdG9yaW5nLXNzbC1pbnRlcm5hbCIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzb21lLW5hbWUtc3NsIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoiYXV0by1jb25maWciLCJjb25maWdNYXAiOnsibmFtZSI6ImF1dG8tbW9uaXRvcmluZy1weGMiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6InZhdWx0LWtleXJpbmctc2VjcmV0Iiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJtb25pdG9yaW5nLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtbW9uaXRvcmluZyIsIm9wdGlvbmFsIjpmYWxzZX19LHsibmFtZSI6Im15c3FsLWluaXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoibW9uaXRvcmluZy1teXNxbC1pbml0Iiwib3B0aW9uYWwiOnRydWV9fV0sImluaXRDb250YWluZXJzIjpbeyJuYW1lIjoicHhjLWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yMjEzLTZjMDhlYTcxIiwiY29tbWFuZCI6WyIvcHhjLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiZGF0YWRpciIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifV0sImNvbnRhaW5lcnMiOlt7Im5hbWUiOiJwbW0tY2xpZW50IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BtbS1jbGllbnQ6ZGV2LWxhdGVzdCIsInBvcnRzIjpbeyJjb250YWluZXJQb3J0Ijo3Nzc3fSx7ImNvbnRhaW5lclBvcnQiOjMwMTAwfSx7ImNvbnRhaW5lclBvcnQiOjMwMTAxfSx7ImNvbnRhaW5lclBvcnQiOjMwMTAyfSx7ImNvbnRhaW5lclBvcnQiOjMwMTAzfSx7ImNvbnRhaW5lclBvcnQiOjMwMTA0fSx7ImNvbnRhaW5lclBvcnQiOjMwMTA1fV0sImVudkZyb20iOlt7InNlY3JldFJlZiI6eyJuYW1lIjoibXktZW52LXZhci1zZWNyZXRzIiwib3B0aW9uYWwiOnRydWV9fV0sImVudiI6W3sibmFtZSI6IlBNTV9TRVJWRVIiLCJ2YWx1ZSI6Im1vbml0b3Jpbmctc2VydmljZSJ9LHsibmFtZSI6IlBNTV9VU0VSIiwidmFsdWUiOiJhcGlfa2V5In0seyJuYW1lIjoiUE1NX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtbW9uaXRvcmluZyIsImtleSI6InBtbXNlcnZlcmtleSJ9fX0seyJuYW1lIjoiQ0xJRU5UX1BPUlRfTElTVEVOIiwidmFsdWUiOiI3Nzc3In0seyJuYW1lIjoiQ0xJRU5UX1BPUlRfTUlOIiwidmFsdWUiOiIzMDEwMCJ9LHsibmFtZSI6IkNMSUVOVF9QT1JUX01BWCIsInZhbHVlIjoiMzAxMDUifSx7Im5hbWUiOiJQT0RfTkFNRSIsInZhbHVlRnJvbSI6eyJmaWVsZFJlZiI6eyJmaWVsZFBhdGgiOiJtZXRhZGF0YS5uYW1lIn19fSx7Im5hbWUiOiJQT0RfTkFNRVNQQVNFIiwidmFsdWVGcm9tIjp7ImZpZWxkUmVmIjp7ImZpZWxkUGF0aCI6Im1ldGFkYXRhLm5hbWVzcGFjZSJ9fX0seyJuYW1lIjoiUE1NX0FHRU5UX1NFUlZFUl9BRERSRVNTIiwidmFsdWUiOiJtb25pdG9yaW5nLXNlcnZpY2UifSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX1VTRVJOQU1FIiwidmFsdWUiOiJhcGlfa2V5In0seyJuYW1lIjoiUE1NX0FHRU5UX1NFUlZFUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLW1vbml0b3JpbmciLCJrZXkiOiJwbW1zZXJ2ZXJrZXkifX19LHsibmFtZSI6IlBNTV9BR0VOVF9MSVNURU5fUE9SVCIsInZhbHVlIjoiNzc3NyJ9LHsibmFtZSI6IlBNTV9BR0VOVF9QT1JUU19NSU4iLCJ2YWx1ZSI6IjMwMTAwIn0seyJuYW1lIjoiUE1NX0FHRU5UX1BPUlRTX01BWCIsInZhbHVlIjoiMzAxMDUifSx7Im5hbWUiOiJQTU1fQUdFTlRfQ09ORklHX0ZJTEUiLCJ2YWx1ZSI6Ii91c3IvbG9jYWwvcGVyY29uYS9wbW0yL2NvbmZpZy9wbW0tYWdlbnQueWFtbCJ9LHsibmFtZSI6IlBNTV9BR0VOVF9T"...,
- 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vc3NsLWhhc2giOiJmMTg2NTgyZDNmMDRhOTlkZTMzZTNhOWUzM2U0ODFjNiIsInBlcmNvbmEuY29tL3NzbC1pbnRlcm5hbC1oYXNoIjoiYTk3Mjg3MTkxYjFhOGY5MjJjYTNjNGRlN2YzZTdhMTkifX0sInNwZWMiOnsidm9sdW1lcyI6W3sibmFtZSI6InRtcCIsImVtcHR5RGlyIjp7fX0seyJuYW1lIjoiY29uZmlnIiwiY29uZmlnTWFwIjp7Im5hbWUiOiJtb25pdG9yaW5nLXB4YyIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJtb25pdG9yaW5nLXNzbC1pbnRlcm5hbCIsIm9wdGlvbmFsIjp0cnVlfX0seyJuYW1lIjoic3NsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJzb21lLW5hbWUtc3NsIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoiYXV0by1jb25maWciLCJjb25maWdNYXAiOnsibmFtZSI6ImF1dG8tbW9uaXRvcmluZy1weGMiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6InZhdWx0LWtleXJpbmctc2VjcmV0Iiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJtb25pdG9yaW5nLXZhdWx0Iiwib3B0aW9uYWwiOnRydWV9fSx7Im5hbWUiOiJteXNxbC11c2Vycy1zZWNyZXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiaW50ZXJuYWwtbW9uaXRvcmluZyIsIm9wdGlvbmFsIjpmYWxzZX19LHsibmFtZSI6Im15c3FsLWluaXQtZmlsZSIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoibW9uaXRvcmluZy1teXNxbC1pbml0Iiwib3B0aW9uYWwiOnRydWV9fV0sImluaXRDb250YWluZXJzIjpbeyJuYW1lIjoicHhjLWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yMjEzLTZjMDhlYTcxIiwiY29tbWFuZCI6WyIvcHhjLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiZGF0YWRpciIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifV0sImNvbnRhaW5lcnMiOlt7Im5hbWUiOiJwbW0tY2xpZW50IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BtbS1jbGllbnQ6ZGV2LWxhdGVzdCIsInBvcnRzIjpbeyJjb250YWluZXJQb3J0Ijo3Nzc3fSx7ImNvbnRhaW5lclBvcnQiOjMwMTAwfSx7ImNvbnRhaW5lclBvcnQiOjMwMTAxfSx7ImNvbnRhaW5lclBvcnQiOjMwMTAyfSx7ImNvbnRhaW5lclBvcnQiOjMwMTAzfSx7ImNvbnRhaW5lclBvcnQiOjMwMTA0fSx7ImNvbnRhaW5lclBvcnQiOjMwMTA1fV0sImVudkZyb20iOlt7InNlY3JldFJlZiI6eyJuYW1lIjoibXktZW52LXZhci1zZWNyZXRzIiwib3B0aW9uYWwiOnRydWV9fV0sImVudiI6W3sibmFtZSI6IlBNTV9TRVJWRVIiLCJ2YWx1ZSI6Im1vbml0b3Jpbmctc2VydmljZSJ9LHsibmFtZSI6IlBNTV9VU0VSIiwidmFsdWUiOiJhZG1pbiJ9LHsibmFtZSI6IlBNTV9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLW1vbml0b3JpbmciLCJrZXkiOiJwbW1zZXJ2ZXIifX19LHsibmFtZSI6IkNMSUVOVF9QT1JUX0xJU1RFTiIsInZhbHVlIjoiNzc3NyJ9LHsibmFtZSI6IkNMSUVOVF9QT1JUX01JTiIsInZhbHVlIjoiMzAxMDAifSx7Im5hbWUiOiJDTElFTlRfUE9SVF9NQVgiLCJ2YWx1ZSI6IjMwMTA1In0seyJuYW1lIjoiUE9EX05BTUUiLCJ2YWx1ZUZyb20iOnsiZmllbGRSZWYiOnsiZmllbGRQYXRoIjoibWV0YWRhdGEubmFtZSJ9fX0seyJuYW1lIjoiUE9EX05BTUVTUEFTRSIsInZhbHVlRnJvbSI6eyJmaWVsZFJlZiI6eyJmaWVsZFBhdGgiOiJtZXRhZGF0YS5uYW1lc3BhY2UifX19LHsibmFtZSI6IlBNTV9BR0VOVF9TRVJWRVJfQUREUkVTUyIsInZhbHVlIjoibW9uaXRvcmluZy1zZXJ2aWNlIn0seyJuYW1lIjoiUE1NX0FHRU5UX1NFUlZFUl9VU0VSTkFNRSIsInZhbHVlIjoiYWRtaW4ifSx7Im5hbWUiOiJQTU1fQUdFTlRfU0VSVkVSX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtbW9uaXRvcmluZyIsImtleSI6InBtbXNlcnZlciJ9fX0seyJuYW1lIjoiUE1NX0FHRU5UX0xJU1RFTl9QT1JUIiwidmFsdWUiOiI3Nzc3In0seyJuYW1lIjoiUE1NX0FHRU5UX1BPUlRTX01JTiIsInZhbHVlIjoiMzAxMDAifSx7Im5hbWUiOiJQTU1fQUdFTlRfUE9SVFNfTUFYIiwidmFsdWUiOiIzMDEwNSJ9LHsibmFtZSI6IlBNTV9BR0VOVF9DT05GSUdfRklMRSIsInZhbHVlIjoiL3Vzci9sb2NhbC9wZXJjb25hL3BtbTIvY29uZmlnL3BtbS1hZ2VudC55YW1sIn0seyJuYW1lIjoiUE1NX0FHRU5UX1NFUlZFUl9JTlNF"...,
- 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFiYzcwNGYwMmU3NDg3"...,
+ 			"percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJtb25pdG9yaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InB4YyIsInBlcmNvbmEuY29tL2NvbmZpZ3VyYXRpb24taGFzaCI6ImQ0MWQ4Y2Q5OGYwMGIyMDRlOTgwMDk5OGVjZjg0MjdlIiwicGVyY29uYS5jb20vZW52LXNlY3JldC1jb25maWctaGFzaCI6IjViYzQwM2E4ZDQ0MzI0YzFiYzcwNGYwMmU3NDg3"...,
  					"percona.com/ssl-hash":                    "f186582d3f04a99de33e3a9e33e481c6",
  					"percona.com/ssl-internal-hash":           "a97287191b1a8f922ca3c4de7f3e7a19",
+ 							PeriodSeconds:                 0,
- 							PeriodSeconds:                 10,
+ 		PersistentVolumeClaimRetentionPolicy: nil,
- 		PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}",
+ 					Phase:       "",
- 					Phase:       "Pending",
+ 		PodManagementPolicy:                  "",
- 		PodManagementPolicy:                  "OrderedReady",
  									Port:        {IntVal: 7777},
  						Ports: []v1.ContainerPort{
  				PreemptionPolicy:          nil,
  							ProbeHandler:                  {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}},
  							ProbeHandler: v1.ProbeHandler{
+ 								Protocol:      "",
- 								Protocol:      "TCP",
  							Quobyte:       nil,
  						ReadinessProbe:           &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...},
  						ReadinessProbe: &v1.Probe{
+ 		ReadyReplicas:      0,
- 		ReadyReplicas:      2,
- 		ReadyReplicas:      3,
+ 		Replicas:           0,
+ 		Replicas: &0,
  		Replicas: &2,
- 		Replicas:           2,
- 		Replicas: &2,
  		Replicas: &3,
- 		Replicas:           3,
- 		Replicas: &3,
  						ResizePolicy: nil,
  									ResourceFieldRef: nil,
  						Resources:    {Limits: {s"cpu": {i: {...}, Format: "DecimalSI"}, s"memory": {i: {...}, Format: "DecimalSI"}}, Requests: {s"cpu": {i: {...}, s: "308m", Format: "DecimalSI"}, s"memory": {i: {...}, s: "508M", Format: "DecimalSI"}}},
+ 		ResourceVersion:            "",
- 		ResourceVersion:            "1760524045420239012",
- 		ResourceVersion:            "1760524168887679001",
- 		ResourceVersion:            "1760524397409151012",
- 		ResourceVersion:            "1760524502751679001",
- 		ResourceVersion:            "1760524569658463012",
- 		ResourceVersion:            "1760524666798799001",
- 		ResourceVersion:            "1760524719716431012",
- 		ResourceVersion:            "1760524819033247001",
+ 				RestartPolicy:                 "",
- 				RestartPolicy:                 "Always",
- 		RevisionHistoryLimit:                 &10,
+ 		RevisionHistoryLimit:                 nil,
+ 				SchedulerName:         "",
+ 				SchedulerName: "",
- 				SchedulerName:         "default-scheduler",
- 				SchedulerName: "default-scheduler",
+ 									Scheme:      "",
- 									Scheme:      "HTTP",
  									SecretKeyRef: &v1.SecretKeySelector{
  								SecretName:  "internal-monitoring",
  								SecretName:  "monitoring-mysql-init",
  								SecretName:  "monitoring-ssl-internal",
  								SecretName:  "monitoring-vault",
  								SecretName:  "my-env-var-secrets",
  								SecretName:  "some-name-ssl",
  							Secret: &v1.SecretVolumeSource{
  						SecurityContext:          nil,
+ 				SecurityContext:       nil,
- 				SecurityContext:       s"&PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:nil,AppArmorProfile:nil,SupplementalGroupsPolicy:nil,SELinux"...,
  		Selector: &{MatchLabels: {"app.kubernetes.io/component": "haproxy", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},
  		Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "monitoring", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},
  		SelfLink:                   "",
  				ServiceAccountName:            "default",
  		ServiceName:                          "monitoring-haproxy",
  		ServiceName:                          "monitoring-pxc",
  				SetHostnameAsFQDN:         nil,
  				ShareProcessNamespace: nil,
  				Spec: v1.PersistentVolumeClaimSpec{
  			Spec: v1.PodSpec{
  	Spec: v1.StatefulSetSpec{
  						StartupProbe:             nil,
  				Status: v1.PersistentVolumeClaimStatus{
  	Status: v1.StatefulSetStatus{
  					StorageClassName:          nil,
  				Subdomain:             "",
  				Subdomain:     "",
- 				Subresource: "status",
  							SuccessThreshold:              1,
  								TCPSocket: nil,
  		Template: v1.PodTemplateSpec{
  				TerminationGracePeriodSeconds: &30,
  				TerminationGracePeriodSeconds: &600,
  							TerminationGracePeriodSeconds: nil,
+ 						TerminationMessagePath:   "",
- 						TerminationMessagePath:   "/dev/termination-log",
+ 						TerminationMessagePolicy: "",
- 						TerminationMessagePolicy: "File",
  							TimeoutSeconds:      15,
  							TimeoutSeconds:                5,
  							TimeoutSeconds:      5,
- 				Time:       s"2025-10-15 10:25:45 +0000 UTC",
- 				Time:        s"2025-10-15 10:27:25 +0000 UTC",
- 				Time:        s"2025-10-15 10:29:28 +0000 UTC",
- 				Time:       s"2025-10-15 10:32:34 +0000 UTC",
- 				Time:        s"2025-10-15 10:33:17 +0000 UTC",
- 				Time:        s"2025-10-15 10:35:02 +0000 UTC",
- 				Time:       s"2025-10-15 10:35:18 +0000 UTC",
- 				Time:        s"2025-10-15 10:36:09 +0000 UTC",
- 				Time:        s"2025-10-15 10:37:46 +0000 UTC",
- 				Time:       s"2025-10-15 10:37:48 +0000 UTC",
- 				Time:        s"2025-10-15 10:38:39 +0000 UTC",
- 				Time:        s"2025-10-15 10:40:19 +0000 UTC",
  				Tolerations:           nil,
  				Tolerations:   nil,
- 				TopologySpreadConstraints: nil,
+ 				TopologySpreadConstraints: []v1.TopologySpreadConstraint{},
  				TypeMeta:   {},
  	TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"},
+ 		UID:                        "",
- 		UID:                        "020f6a87-1b51-4394-9601-4a26aca15820",
- 		UID:                        "7c20ba57-ac5e-472e-a8c5-c5a7aa37e9ce",
+ 		UpdatedReplicas:    0,
- 		UpdatedReplicas:    2,
- 		UpdatedReplicas:    3,
+ 		UpdateRevision:     "",
- 		UpdateRevision:     "monitoring-haproxy-5fc9ddf657",
- 		UpdateRevision:     "monitoring-haproxy-7c56cc486c",
- 		UpdateRevision:     "monitoring-haproxy-7d88c96f4c",
- 		UpdateRevision:     "monitoring-haproxy-865847fd75",
- 		UpdateRevision:     "monitoring-pxc-65449d488",
- 		UpdateRevision:     "monitoring-pxc-65896bc69d",
- 		UpdateRevision:     "monitoring-pxc-6899cbbd7b",
- 		UpdateRevision:     "monitoring-pxc-866c4cc6fb",
  		UpdateStrategy:                       {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}},
  &v1.StatefulSet{
  								Value: "",
- 								Value:     "admin",
+ 								Value:     "api_key",
  								ValueFrom: nil,
  								ValueFrom: &v1.EnvVarSource{
+ 								Value:     "$(PMM_PREFIX)$(POD_NAMESPASE)-$(POD_NAME)",
- 								Value:     "$(POD_NAMESPASE)-$(POD_NAME)",
  					VolumeAttributesClassName: nil,
  		VolumeClaimTemplates:                 nil,
  		VolumeClaimTemplates: []v1.PersistentVolumeClaim{
  						VolumeDevices: nil,
- 					VolumeMode:                &"Filesystem",
+ 					VolumeMode:                nil,
  						VolumeMounts:  {{Name: "bin", MountPath: "/var/lib/mysql"}},
  						VolumeMounts:  {{Name: "datadir", MountPath: "/var/lib/mysql"}},
  						VolumeMounts:  {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...},
  					VolumeName:                "",
  						VolumeSource: v1.VolumeSource{
  				Volumes: []v1.Volume{
  							VsphereVolume: nil,
  						WorkingDir: "",
+ kubectl get pxc --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl patch pxc -n monitoring-2-0-932 monitoring --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/monitoring patched (no change)
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.hzev8xwL9k
++ mktemp
+ local LAST_ERR=/tmp/tmp.lstahuHUrA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.hzev8xwL9k
perconaxtradbcluster.pxc.percona.com "monitoring" deleted from monitoring-2-0-932 namespace
+ cat /tmp/tmp.lstahuHUrA
+ rm /tmp/tmp.hzev8xwL9k /tmp/tmp.lstahuHUrA
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.87l7WmhHpG
++ mktemp
+ local LAST_ERR=/tmp/tmp.AYXmuxbWUX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.87l7WmhHpG
No resources found
+ cat /tmp/tmp.AYXmuxbWUX
+ rm /tmp/tmp.87l7WmhHpG /tmp/tmp.AYXmuxbWUX
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.9LvA2YSteD
++ mktemp
+ local LAST_ERR=/tmp/tmp.sR1IiGFiHA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.9LvA2YSteD
No resources found
+ cat /tmp/tmp.sR1IiGFiHA
+ rm /tmp/tmp.9LvA2YSteD /tmp/tmp.sR1IiGFiHA
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.5r3FfdO8K5
++ mktemp
+ local LAST_ERR=/tmp/tmp.9T0SAufGfF
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5r3FfdO8K5
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.9T0SAufGfF
+ rm /tmp/tmp.5r3FfdO8K5 /tmp/tmp.9T0SAufGfF
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ rm -rf /tmp/tmp.h65oP78O1Z
+ kubectl_bin delete --grace-period=0 --force=true namespace monitoring-2-0-932
++ mktemp
+ local LAST_OUT=/tmp/tmp.XPCeQ0YXNu
++ mktemp
+ local LAST_ERR=/tmp/tmp.MlEuiOsKye
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace monitoring-2-0-932
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.0q0Txc6KBM
++ mktemp
+ local LAST_ERR=/tmp/tmp.wRJbbETK4f
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator