Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/logs/monitoring-pmm3-8-0.log
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ cluster=monitoring
+ create_infra monitoring-pmm3-25371
+ local ns=monitoring-pmm3-25371
+ '[' -n pxc-operator ']'
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ kubectl patch pxc -n monitoring-pmm3-32079 monitoring --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/monitoring patched (no change)
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.lAU4RnoP4l
++ mktemp
+ local LAST_ERR=/tmp/tmp.PP9ewObi5p
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lAU4RnoP4l
perconaxtradbcluster.pxc.percona.com "monitoring" deleted from monitoring-pmm3-32079 namespace
+ cat /tmp/tmp.PP9ewObi5p
+ rm /tmp/tmp.lAU4RnoP4l /tmp/tmp.PP9ewObi5p
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.TeFhLaafRU
++ mktemp
+ local LAST_ERR=/tmp/tmp.T2biXKJHTy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.TeFhLaafRU
No resources found
+ cat /tmp/tmp.T2biXKJHTy
+ rm /tmp/tmp.TeFhLaafRU /tmp/tmp.T2biXKJHTy
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.O1yaEAdnLa
++ mktemp
+ local LAST_ERR=/tmp/tmp.5rdacxWOxj
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.O1yaEAdnLa
No resources found
+ cat /tmp/tmp.5rdacxWOxj
+ rm /tmp/tmp.O1yaEAdnLa /tmp/tmp.5rdacxWOxj
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
++ tail -n1
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ awk '{print $1}'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ grep chaos-mesh.org
++ awk '{print $1}'
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrolebinding
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


+ awk '{print$1}'
-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.tyvjjiu2K5
++ mktemp
+ xargs kubectl delete ns
+ local LAST_ERR=/tmp/tmp.KjhjcGoU8y
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.a9TArwL46k
++ mktemp
+ local LAST_ERR=/tmp/tmp.mO9EJvivts
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.a9TArwL46k
+ cat /tmp/tmp.mO9EJvivts
+ rm /tmp/tmp.a9TArwL46k /tmp/tmp.mO9EJvivts
+ return 0
namespace "monitoring-pmm3-32079" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.tyvjjiu2K5
namespace "pxc-operator" deleted
+ cat /tmp/tmp.KjhjcGoU8y
+ rm /tmp/tmp.tyvjjiu2K5 /tmp/tmp.KjhjcGoU8y
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.1eAIAxm8p7
++ mktemp
+ local LAST_ERR=/tmp/tmp.FAFNEgvB6i
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1eAIAxm8p7
namespace/pxc-operator created
+ cat /tmp/tmp.FAFNEgvB6i
+ rm /tmp/tmp.1eAIAxm8p7 /tmp/tmp.FAFNEgvB6i
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.izAdU2aCF4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SSbW693FTt
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.izAdU2aCF4
++ cat /tmp/tmp.SSbW693FTt
++ rm /tmp/tmp.izAdU2aCF4 /tmp/tmp.SSbW693FTt
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2002-42929599-9-cluster8 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.vbgWNDKWrR
++ mktemp
+ local LAST_ERR=/tmp/tmp.NatAkIKv3M
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2002-42929599-9-cluster8 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vbgWNDKWrR
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2002-42929599-9-cluster8" modified.
+ cat /tmp/tmp.NatAkIKv3M
+ rm /tmp/tmp.vbgWNDKWrR /tmp/tmp.NatAkIKv3M
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.i0U4BShS8f
++ mktemp
+ local LAST_ERR=/tmp/tmp.5qpeNtoNDW
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.i0U4BShS8f
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.5qpeNtoNDW
+ rm /tmp/tmp.i0U4BShS8f /tmp/tmp.5qpeNtoNDW
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ kubectl_bin apply -f -
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/deploy/cw-rbac.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.PnGa0KTH9V
++ mktemp
+ local LAST_ERR=/tmp/tmp.ay4Y7hhlsA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.PnGa0KTH9V
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.ay4Y7hhlsA
+ rm /tmp/tmp.PnGa0KTH9V /tmp/tmp.ay4Y7hhlsA
+ return 0
+ kubectl_bin apply -f -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2002-42929599^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.eDMs5jGQxX
++ mktemp
+ local LAST_ERR=/tmp/tmp.mUEYY0NmO3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.eDMs5jGQxX
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.mUEYY0NmO3
+ rm /tmp/tmp.eDMs5jGQxX /tmp/tmp.mUEYY0NmO3
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.MSVXOhB9hG
++ mktemp
+ local LAST_ERR=/tmp/tmp.yR1udh3339
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MSVXOhB9hG
pod/percona-xtradb-cluster-operator-6c8dc776-l2p49 condition met
+ cat /tmp/tmp.yR1udh3339
+ rm /tmp/tmp.MSVXOhB9hG /tmp/tmp.yR1udh3339
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iubNQhg9jP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FxBzh9Ium0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.iubNQhg9jP
++ cat /tmp/tmp.FxBzh9Ium0
++ rm /tmp/tmp.iubNQhg9jP /tmp/tmp.FxBzh9Ium0
++ return 0
+ wait_pod percona-xtradb-cluster-operator-6c8dc776-l2p49 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-6c8dc776-l2p49
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-6c8dc776-l2p49
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-6c8dc776-l2p49 condition met
waiting for pod/percona-xtradb-cluster-operator-6c8dc776-l2p49 to become Ready.Ok
+ sleep 3
+ create_namespace monitoring-pmm3-25371
+ local namespace=monitoring-pmm3-25371
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ tail -n1
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl api-resources
++ awk '{print $1}'
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces monitoring-pmm3-25371'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces monitoring-pmm3-25371
-----------------------------------------------------------------------------------

+ awk '{print$1}'
+ kubectl_bin delete namespace monitoring-pmm3-25371
+ xargs kubectl delete ns
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.OMCsQp7j1A
++ mktemp
+ local LAST_ERR=/tmp/tmp.X7gxgXZjmr
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.jRKdBSyq9Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.hpRZYwmMOy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace monitoring-pmm3-25371
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.OMCsQp7j1A
+ cat /tmp/tmp.X7gxgXZjmr
+ rm /tmp/tmp.OMCsQp7j1A /tmp/tmp.X7gxgXZjmr
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace monitoring-pmm3-25371
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace monitoring-pmm3-25371
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.jRKdBSyq9Q
+ cat /tmp/tmp.hpRZYwmMOy
Error from server (NotFound): namespaces "monitoring-pmm3-25371" not found
+ rm /tmp/tmp.jRKdBSyq9Q /tmp/tmp.hpRZYwmMOy
+ return 1
+ :
+ wait_for_delete namespace/monitoring-pmm3-25371
+ local res=namespace/monitoring-pmm3-25371
+ echo -n 'waiting for namespace/monitoring-pmm3-25371 to be deleted'
waiting for namespace/monitoring-pmm3-25371 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "monitoring-pmm3-25371" not found
+ desc 'create namespace monitoring-pmm3-25371'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace monitoring-pmm3-25371
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace monitoring-pmm3-25371
++ mktemp
+ local LAST_OUT=/tmp/tmp.cNWPFZBzRA
++ mktemp
+ local LAST_ERR=/tmp/tmp.inYP4T8G6C
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace monitoring-pmm3-25371
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.cNWPFZBzRA
namespace/monitoring-pmm3-25371 created
+ cat /tmp/tmp.inYP4T8G6C
+ rm /tmp/tmp.cNWPFZBzRA /tmp/tmp.inYP4T8G6C
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6Qqoq3DoXO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rZEUhfASUd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6Qqoq3DoXO
++ cat /tmp/tmp.rZEUhfASUd
++ rm /tmp/tmp.6Qqoq3DoXO /tmp/tmp.rZEUhfASUd
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2002-42929599-9-cluster8 --namespace=monitoring-pmm3-25371
++ mktemp
+ local LAST_OUT=/tmp/tmp.TmWKFZCQ5Y
++ mktemp
+ local LAST_ERR=/tmp/tmp.piPeGl1DRw
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2002-42929599-9-cluster8 --namespace=monitoring-pmm3-25371
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.TmWKFZCQ5Y
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2002-42929599-9-cluster8" modified.
+ cat /tmp/tmp.piPeGl1DRw
+ rm /tmp/tmp.TmWKFZCQ5Y /tmp/tmp.piPeGl1DRw
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ApQHkJGnIs
++ mktemp
+ local LAST_ERR=/tmp/tmp.uURjOwiUZ1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ApQHkJGnIs
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.uURjOwiUZ1
+ rm /tmp/tmp.ApQHkJGnIs /tmp/tmp.uURjOwiUZ1
+ return 0
+ deploy_helm monitoring-pmm3-25371
+ helm repo add hashicorp https://helm.releases.hashicorp.com
"hashicorp" already exists with the same configuration, skipping
+ helm repo add minio https://charts.min.io/
"minio" already exists with the same configuration, skipping
+ helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "chaos-mesh" chart repository
...Successfully got an update from the "minio" chart repository
...Successfully got an update from the "percona" chart repository
...Successfully got an update from the "hashicorp" chart repository
Update Complete. ⎈Happy Helming!⎈
+ desc 'install PMM Server'
+ set +o xtrace


-----------------------------------------------------------------------------------
install PMM Server
-----------------------------------------------------------------------------------

+ deploy_pmm3_server
+ helm uninstall -n '' monitoring
Error: uninstall: Release not loaded: monitoring: release: not found
+ :
+ helm repo remove percona
"percona" has been removed from your repositories
+ kubectl delete clusterrole monitoring --ignore-not-found
+ kubectl delete clusterrolebinding monitoring --ignore-not-found
+ helm repo add percona https://percona.github.io/percona-helm-charts/
"percona" has been added to your repositories
+ helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "minio" chart repository
...Successfully got an update from the "chaos-mesh" chart repository
...Successfully got an update from the "percona" chart repository
...Successfully got an update from the "hashicorp" chart repository
Update Complete. ⎈Happy Helming!⎈
+ '[' '!' -z '' ']'
+ retry 10 60 helm install monitoring percona/pmm -n '' --set fullnameOverride=monitoring --set image.tag=3.1.0 --set image.repository=perconalab/pmm-server --set service.type=LoadBalancer --force
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install monitoring percona/pmm -n '' --set fullnameOverride=monitoring --set image.tag=3.1.0 --set image.repository=perconalab/pmm-server --set service.type=LoadBalancer --force
NAME: monitoring
LAST DEPLOYED: Tue Nov  4 08:47:56 2025
NAMESPACE: monitoring-pmm3-25371
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
Percona Monitoring and Management (PMM)

An open source database monitoring, observability and management tool
Check more info here: https://docs.percona.com/percona-monitoring-and-management/index.html

Get the application URL:
     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
           You can watch the status of by running 'kubectl get --namespace monitoring-pmm3-25371 svc -w monitoring-service'
  export SERVICE_IP=$(kubectl get svc --namespace monitoring-pmm3-25371 monitoring-service -o jsonpath="{.status.loadBalancer.ingress[0].ip}")
  echo https://$SERVICE_IP:

Get password for the "admin" user:
  export ADMIN_PASS=$(kubectl get secret pmm-secret --namespace monitoring-pmm3-25371 -o jsonpath='{.data.PMM_ADMIN_PASSWORD}' | base64 --decode)
  echo $ADMIN_PASS
+ kubectl_bin wait --for=condition=Ready pod/monitoring-0 --timeout=120s
++ mktemp
+ local LAST_OUT=/tmp/tmp.AdvDRyw5sH
++ mktemp
+ local LAST_ERR=/tmp/tmp.8koHnRV3RV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod/monitoring-0 --timeout=120s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.AdvDRyw5sH
pod/monitoring-0 condition met
+ cat /tmp/tmp.8koHnRV3RV
+ rm /tmp/tmp.AdvDRyw5sH /tmp/tmp.8koHnRV3RV
+ return 0
+ kubectl_bin exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
++ mktemp
+ local LAST_OUT=/tmp/tmp.UKnHyEHCg9
++ mktemp
+ local LAST_ERR=/tmp/tmp.O5WwgiZ5Y1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec monitoring-0 -- bash -c 'ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UKnHyEHCg9
+ cat /tmp/tmp.O5WwgiZ5Y1
+ rm /tmp/tmp.UKnHyEHCg9 /tmp/tmp.O5WwgiZ5Y1
+ return 0
+ desc 'create secret'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secret
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/conf/secrets.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.q2s4cjDlTf
++ mktemp
+ local LAST_ERR=/tmp/tmp.Nv3zR3Dnmn
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/conf/secrets.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.q2s4cjDlTf
secret/my-cluster-secrets created
+ cat /tmp/tmp.Nv3zR3Dnmn
+ rm /tmp/tmp.q2s4cjDlTf /tmp/tmp.Nv3zR3Dnmn
+ return 0
+ desc 'add PMM3 token to secret'
+ set +o xtrace


-----------------------------------------------------------------------------------
add PMM3 token to secret
-----------------------------------------------------------------------------------

++ get_pmm_server_token operator
++ local key_name=operator
++ [[ -z operator ]]
++ local ADMIN_PASSWORD
+++ kubectl get secret pmm-secret -o 'jsonpath={.data.PMM_ADMIN_PASSWORD}'
+++ base64 --decode
++ ADMIN_PASSWORD='%m14P[&H=L4`[$b_'
++ [[ -z %m14P[&H=L4`[$b_ ]]
++ local create_response create_status_code create_json_response
++ local retry=0
++ [[ '' == 201 ]]
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
++++ grep -q NotFound
+++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.ekI2p5Mi8v
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.FTQNZGRvKb
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.ekI2p5Mi8v
+++++ cat /tmp/tmp.FTQNZGRvKb
+++++ rm /tmp/tmp.ekI2p5Mi8v /tmp/tmp.FTQNZGRvKb
+++++ return 0
++++ '[' LoadBalancer = ClusterIP ']'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}'
++++ egrep -q 'hostname|ip'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.f8jTRLbQZJ
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.luPinOmp6C
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.f8jTRLbQZJ
++++ cat /tmp/tmp.luPinOmp6C
++++ rm /tmp/tmp.f8jTRLbQZJ /tmp/tmp.luPinOmp6C
++++ return 0
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.9YVLzsOKIT
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.4JR0JS74QD
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.9YVLzsOKIT
++++ cat /tmp/tmp.4JR0JS74QD
++++ rm /tmp/tmp.9YVLzsOKIT /tmp/tmp.4JR0JS74QD
++++ return 0
+++ curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"name":"operator", "role":"Admin", "isDisabled":false}' --user 'admin:%m14P[&H=L4`[$b_' https://104.197.43.97/graph/api/serviceaccounts -w '\n%{http_code}'
++ create_response='{"id":2,"name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}
201'
+++ echo '{"id":2,"name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}
201'
+++ tail -n1
++ create_status_code=201
+++ echo '{"id":2,"name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}
201'
+++ sed '$ d'
++ create_json_response='{"id":2,"name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}'
++ sleep 5
++ let retry+=1
++ '[' 1 -ge 24 ']'
++ [[ 201 == 201 ]]
++ local service_account_id
+++ echo '{"id":2,"name":"operator","login":"sa-1-operator","orgId":1,"isDisabled":false,"role":"Admin","tokens":0,"avatarUrl":""}'
+++ jq -r .id
++ service_account_id=2
++ [[ -z 2 ]]
++ [[ 2 == \n\u\l\l ]]
++ local token_response token_status_code token_json_response
++++ get_service_ip monitoring-service
++++ local service=monitoring-service
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
++++ grep -q NotFound
+++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.spec.type}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.kzWbR8lRNq
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.JeVa3PStrN
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get service/monitoring-service -o 'jsonpath={.spec.type}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.kzWbR8lRNq
+++++ cat /tmp/tmp.JeVa3PStrN
+++++ rm /tmp/tmp.kzWbR8lRNq /tmp/tmp.JeVa3PStrN
+++++ return 0
++++ '[' LoadBalancer = ClusterIP ']'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[]}'
++++ egrep -q 'hostname|ip'
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.gl3Az4HKBU
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.tiLB2OC0SL
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.gl3Az4HKBU
++++ cat /tmp/tmp.tiLB2OC0SL
++++ rm /tmp/tmp.gl3Az4HKBU /tmp/tmp.tiLB2OC0SL
++++ return 0
++++ kubectl_bin get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.Nsdn8wKsPF
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.4xWNbKHDQH
++++ local exit_status=0
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl get service/monitoring-service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 ']'
++++ break
++++ cat /tmp/tmp.Nsdn8wKsPF
++++ cat /tmp/tmp.4xWNbKHDQH
++++ rm /tmp/tmp.Nsdn8wKsPF /tmp/tmp.4xWNbKHDQH
++++ return 0
+++ curl --insecure -s -X POST -H 'Content-Type: application/json' -d '{"name":"operator"}' --user 'admin:%m14P[&H=L4`[$b_' https://104.197.43.97/graph/api/serviceaccounts/2/tokens -w '\n%{http_code}'
++ token_response='{"id":1,"name":"operator","key":"glsa_aqofv1XGxuiZhUX2hPGDHqKMKsNlCM2Y_1cef1706"}
200'
+++ echo '{"id":1,"name":"operator","key":"glsa_aqofv1XGxuiZhUX2hPGDHqKMKsNlCM2Y_1cef1706"}
+++ tail -n1
200'
++ token_status_code=200
+++ echo '{"id":1,"name":"operator","key":"glsa_aqofv1XGxuiZhUX2hPGDHqKMKsNlCM2Y_1cef1706"}
200'
+++ sed '$ d'
++ token_json_response='{"id":1,"name":"operator","key":"glsa_aqofv1XGxuiZhUX2hPGDHqKMKsNlCM2Y_1cef1706"}'
++ [[ 200 -ne 200 ]]
++ jq -r .key
++ echo '{"id":1,"name":"operator","key":"glsa_aqofv1XGxuiZhUX2hPGDHqKMKsNlCM2Y_1cef1706"}'
+ TOKEN=glsa_aqofv1XGxuiZhUX2hPGDHqKMKsNlCM2Y_1cef1706
+ kubectl_bin patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmservertoken": "glsa_aqofv1XGxuiZhUX2hPGDHqKMKsNlCM2Y_1cef1706"}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.lgaWSHdbVg
++ mktemp
+ local LAST_ERR=/tmp/tmp.OLDciOnrtV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmservertoken": "glsa_aqofv1XGxuiZhUX2hPGDHqKMKsNlCM2Y_1cef1706"}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lgaWSHdbVg
secret/my-cluster-secrets patched
+ cat /tmp/tmp.OLDciOnrtV
+ rm /tmp/tmp.lgaWSHdbVg /tmp/tmp.OLDciOnrtV
+ return 0
+ desc 'create PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC cluster
-----------------------------------------------------------------------------------

+ spinup_pxc monitoring /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/conf/monitoring.yml 3 120
+ local cluster=monitoring
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/conf/monitoring.yml
+ local size=3
+ local sleep=120
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/conf/client.yml
+ local LAST_OUT=/tmp/tmp.gQwNtOXMMp
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ local LAST_ERR=/tmp/tmp.vmzBQ5Hrwp
+ local exit_status=0
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.monitoring-pmm3-25371~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2002-42929599#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gQwNtOXMMp
deployment.apps/pxc-client created
+ cat /tmp/tmp.vmzBQ5Hrwp
+ rm /tmp/tmp.gQwNtOXMMp /tmp/tmp.vmzBQ5Hrwp
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/conf/monitoring.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/conf/monitoring.yml
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.monitoring-pmm3-25371~
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2002-42929599#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/conf/monitoring.yml
+ kubectl_bin apply -f -
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ local LAST_OUT=/tmp/tmp.nbuzbZSgYP
++ mktemp
+ local LAST_ERR=/tmp/tmp.2U37P5LnUZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nbuzbZSgYP
perconaxtradbcluster.pxc.percona.com/monitoring created
+ cat /tmp/tmp.2U37P5LnUZ
+ rm /tmp/tmp.nbuzbZSgYP /tmp/tmp.2U37P5LnUZ
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy monitoring
++ local target_cluster=monitoring
+++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ncQKFMPJzx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ZpBpQZJDGy
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.ncQKFMPJzx
+++ cat /tmp/tmp.ZpBpQZJDGy
+++ rm /tmp/tmp.ncQKFMPJzx /tmp/tmp.ZpBpQZJDGy
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo monitoring-haproxy
++ return
+ local proxy=monitoring-haproxy
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n monitoring-pmm3-25371
++ mktemp
+ local LAST_OUT=/tmp/tmp.UEYK2iQh7W
++ mktemp
+ local LAST_ERR=/tmp/tmp.w0xptPNa6M
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n monitoring-pmm3-25371
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n monitoring-pmm3-25371
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n monitoring-pmm3-25371
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UEYK2iQh7W
pod/monitoring-haproxy-0 condition met
pod/monitoring-pxc-0 condition met
+ cat /tmp/tmp.w0xptPNa6M
+ rm /tmp/tmp.UEYK2iQh7W /tmp/tmp.w0xptPNa6M
+ return 0
+ wait_for_running monitoring-haproxy 1
+ local name=monitoring-haproxy
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod monitoring-haproxy-0 480
+ local pod=monitoring-haproxy-0
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo monitoring-haproxy-0
+ local container=
+ set +o xtrace
pod/monitoring-haproxy-0 condition met
waiting for pod/monitoring-haproxy-0 to become Ready.Ok
+ wait_for_running monitoring-pxc 3
+ local name=monitoring-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod monitoring-pxc-0 480
+ local pod=monitoring-pxc-0
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
++ echo monitoring-pxc-0
+ local container=pxc
+ set +o xtrace
pod/monitoring-pxc-0 condition met
waiting for pod/monitoring-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod monitoring-pxc-1 480
+ local pod=monitoring-pxc-1
+ local max_retry=480
+ local ns=
++ echo monitoring-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/monitoring-pxc-1 condition met
waiting for pod/monitoring-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod monitoring-pxc-2 480
+ local pod=monitoring-pxc-2
+ local max_retry=480
+ local ns=
++ echo monitoring-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/monitoring-pxc-2 condition met
waiting for pod/monitoring-pxc-2 to become Ready.Ok
+ sleep 120
++ kubectl get pxc monitoring -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2ayNOHKmud
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fYcMz5bkoB
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2ayNOHKmud
++ cat /tmp/tmp.fYcMz5bkoB
++ rm /tmp/tmp.2ayNOHKmud /tmp/tmp.fYcMz5bkoB
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h monitoring-haproxy -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h monitoring-haproxy -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zOKcO8so0o
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qwEaSNTC1p
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zOKcO8so0o
++ cat /tmp/tmp.qwEaSNTC1p
++ rm /tmp/tmp.zOKcO8so0o /tmp/tmp.qwEaSNTC1p
++ return 0
+ client_pod=pxc-client-59944c5bbf-jf5g9
+ wait_pod pxc-client-59944c5bbf-jf5g9
+ local pod=pxc-client-59944c5bbf-jf5g9
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-jf5g9
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-jf5g9 condition met
waiting for pod/pxc-client-59944c5bbf-jf5g9 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h monitoring-haproxy -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h monitoring-haproxy -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VcomSAUFdk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.frhxnGeO4C
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VcomSAUFdk
++ cat /tmp/tmp.frhxnGeO4C
++ rm /tmp/tmp.VcomSAUFdk /tmp/tmp.frhxnGeO4C
++ return 0
+ client_pod=pxc-client-59944c5bbf-jf5g9
+ wait_pod pxc-client-59944c5bbf-jf5g9
+ local pod=pxc-client-59944c5bbf-jf5g9
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-jf5g9
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-jf5g9 condition met
waiting for pod/pxc-client-59944c5bbf-jf5g9 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h monitoring-pxc-0.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rcq5DxdMy1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MTEwa3XfWJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rcq5DxdMy1
++ cat /tmp/tmp.MTEwa3XfWJ
++ rm /tmp/tmp.rcq5DxdMy1 /tmp/tmp.MTEwa3XfWJ
++ return 0
+ client_pod=pxc-client-59944c5bbf-jf5g9
+ wait_pod pxc-client-59944c5bbf-jf5g9
+ local pod=pxc-client-59944c5bbf-jf5g9
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-jf5g9
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-jf5g9 condition met
waiting for pod/pxc-client-59944c5bbf-jf5g9 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.Zez20BwD9u/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/select-1.sql /tmp/tmp.Zez20BwD9u/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h monitoring-pxc-1.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h monitoring-pxc-1.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h monitoring-pxc-1.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h monitoring-pxc-1.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8ulqvstgMN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.baLwCBHMAY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8ulqvstgMN
++ cat /tmp/tmp.baLwCBHMAY
++ rm /tmp/tmp.8ulqvstgMN /tmp/tmp.baLwCBHMAY
++ return 0
+ client_pod=pxc-client-59944c5bbf-jf5g9
+ wait_pod pxc-client-59944c5bbf-jf5g9
+ local pod=pxc-client-59944c5bbf-jf5g9
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-jf5g9
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-jf5g9 condition met
waiting for pod/pxc-client-59944c5bbf-jf5g9 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.Zez20BwD9u/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/select-1.sql /tmp/tmp.Zez20BwD9u/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h monitoring-pxc-2.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h monitoring-pxc-2.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h monitoring-pxc-2.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h monitoring-pxc-2.monitoring-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.o1hqJU2UBM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ahtafl8gFL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.o1hqJU2UBM
++ cat /tmp/tmp.ahtafl8gFL
++ rm /tmp/tmp.o1hqJU2UBM /tmp/tmp.ahtafl8gFL
++ return 0
+ client_pod=pxc-client-59944c5bbf-jf5g9
+ wait_pod pxc-client-59944c5bbf-jf5g9
+ local pod=pxc-client-59944c5bbf-jf5g9
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-jf5g9
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-jf5g9 condition met
waiting for pod/pxc-client-59944c5bbf-jf5g9 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.Zez20BwD9u/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/select-1.sql /tmp/tmp.Zez20BwD9u/select-1.sql
++ is_keyring_plugin_in_use monitoring
++ local cluster=monitoring
++ kubectl_bin exec -it monitoring-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Wu5u0TZerL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4VlNnouHIc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it monitoring-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Wu5u0TZerL
++ cat /tmp/tmp.4VlNnouHIc
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.Wu5u0TZerL /tmp/tmp.4VlNnouHIc
++ return 0
+ '[' '' ']'
+ wait_for_generation sts/monitoring-pxc 1
+ local resource=sts/monitoring-pxc
+ local target_generation=1
+ echo 'Waiting for sts/monitoring-pxc to reach generation 1...'
Waiting for sts/monitoring-pxc to reach generation 1...
+ true
++ kubectl get sts/monitoring-pxc -o 'jsonpath={.metadata.generation}'
+ current_generation=1
+ '[' 1 -eq 1 ']'
+ echo 'Resource sts/monitoring-pxc has reached generation 1.'
Resource sts/monitoring-pxc has reached generation 1.
+ break
+ wait_for_generation sts/monitoring-haproxy 1
+ local resource=sts/monitoring-haproxy
+ local target_generation=1
+ echo 'Waiting for sts/monitoring-haproxy to reach generation 1...'
Waiting for sts/monitoring-haproxy to reach generation 1...
+ true
++ kubectl get sts/monitoring-haproxy -o 'jsonpath={.metadata.generation}'
+ current_generation=1
+ '[' 1 -eq 1 ']'
+ echo 'Resource sts/monitoring-haproxy has reached generation 1.'
Resource sts/monitoring-haproxy has reached generation 1.
+ break
+ sleep 10
+ kubectl wait pod -l app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --for=condition=ready --timeout=600s
pod/monitoring-haproxy-0 condition met
pod/monitoring-haproxy-1 condition met
pod/monitoring-pxc-0 condition met
pod/monitoring-pxc-1 condition met
pod/monitoring-pxc-2 condition met
+ wait_cluster_consistency monitoring 3 2
+ local cluster_name=monitoring
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/monitoring to be ready'
waiting for pxc/monitoring to be ready++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qwuNUONYQI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DCLjbgPnTt
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.qwuNUONYQI
++ cat /tmp/tmp.DCLjbgPnTt
++ rm /tmp/tmp.qwuNUONYQI /tmp/tmp.DCLjbgPnTt
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NuFN9p7rWM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VtiJibyzpT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NuFN9p7rWM
++ cat /tmp/tmp.VtiJibyzpT
++ rm /tmp/tmp.NuFN9p7rWM /tmp/tmp.VtiJibyzpT
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine monitoring
+++ local cluster_name=monitoring
++++ get_proxy monitoring
++++ local target_cluster=monitoring
+++++ kubectl_bin get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.psg6ZkxKqp
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.cu2St7lDQX
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc monitoring -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.psg6ZkxKqp
+++++ cat /tmp/tmp.cu2St7lDQX
+++++ rm /tmp/tmp.psg6ZkxKqp /tmp/tmp.cu2St7lDQX
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo monitoring-haproxy
++++ return
+++ local cluster_proxy=monitoring-haproxy
+++ echo haproxy
++ kubectl_bin get pxc monitoring -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Zic7kKLttT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jt6kYyxxjU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc monitoring -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Zic7kKLttT
++ cat /tmp/tmp.jt6kYyxxjU
++ rm /tmp/tmp.Zic7kKLttT /tmp/tmp.jt6kYyxxjU
++ return 0
+ [[ 2 == \2 ]]
+ echo

+ compare_kubectl statefulset/monitoring-pxc -no-prefix
+ local resource=statefulset/monitoring-pxc
+ local postfix=-no-prefix
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix.yml
+ local new_result=/tmp/tmp.Zez20BwD9u/statefulset_monitoring-pxc.yml
+ desc 'compare statefulset/monitoring-pxc--no-prefix'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/monitoring-pxc--no-prefix
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-80.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ version_gt 1.33
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.33'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k129.yml ']'
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.27'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127.yml ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127.yml
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127-oc.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.31 >= 1.29'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127-k129-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127-eks.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127-aks.yml ']'
+ kubectl_bin get -o yaml statefulset/monitoring-pxc
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("monitoring-pmm3-25371", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.spec.crVersion) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.p6Y9gn5bSa
++ mktemp
+ local LAST_ERR=/tmp/tmp.xU9gmFpIBu
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/monitoring-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.p6Y9gn5bSa
+ cat /tmp/tmp.xU9gmFpIBu
+ rm /tmp/tmp.p6Y9gn5bSa /tmp/tmp.xU9gmFpIBu
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127.yml /tmp/tmp.Zez20BwD9u/statefulset_monitoring-pxc.yml
--- /mnt/jenkins/workspace/cloud-pxc-operator_PR-2002/e2e-tests/monitoring-pmm3/compare/statefulset_monitoring-pxc-no-prefix-k127.yml	2025-11-04 07:54:10.972958671 +0000
+++ /tmp/tmp.Zez20BwD9u/statefulset_monitoring-pxc.yml	2025-11-04 08:57:15.437994036 +0000
@@ -202,6 +202,8 @@
               value: /var/lib/mysql/notify.sock
             - name: MYSQL_STATE_FILE
               value: /var/lib/mysql/mysql.state
+            - name: LD_PRELOAD
+              value: /usr/lib64/libjemalloc.so.1
           envFrom:
             - secretRef:
                 name: my-env-var-secrets