Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/logs/users-8-0.log
WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1
+ create_infra users-29635
+ local ns=users-29635
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n users-31037 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.4bFKwGGcc9
++ mktemp
+ local LAST_ERR=/tmp/tmp.2JX1xGMkZY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.4bFKwGGcc9
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.2JX1xGMkZY
+ rm /tmp/tmp.4bFKwGGcc9 /tmp/tmp.2JX1xGMkZY
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.yqVzM6Ho1O
++ mktemp
+ local LAST_ERR=/tmp/tmp.MwM7D5HPtZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.yqVzM6Ho1O
No resources found
+ cat /tmp/tmp.MwM7D5HPtZ
+ rm /tmp/tmp.yqVzM6Ho1O /tmp/tmp.MwM7D5HPtZ
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.rUhFUk9ACt
++ mktemp
+ local LAST_ERR=/tmp/tmp.B5Zy2jGi8x
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rUhFUk9ACt
No resources found
+ cat /tmp/tmp.B5Zy2jGi8x
+ rm /tmp/tmp.rUhFUk9ACt /tmp/tmp.B5Zy2jGi8x
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.7uN0Jpn7MT
++ mktemp
+ local LAST_OUT=/tmp/tmp.bg6VdLniwS
++ mktemp
+ local LAST_ERR=/tmp/tmp.adRcw5YKW4
+ local exit_status=0
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.ejqAfLdj6j
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7uN0Jpn7MT
+ cat /tmp/tmp.adRcw5YKW4
+ rm /tmp/tmp.7uN0Jpn7MT /tmp/tmp.adRcw5YKW4
+ return 0
namespace "users-31037" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.bg6VdLniwS
namespace "pxc-operator" deleted
+ cat /tmp/tmp.ejqAfLdj6j
+ rm /tmp/tmp.bg6VdLniwS /tmp/tmp.ejqAfLdj6j
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'namespace/pxc-operator - '
namespace/pxc-operator - + set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.GEbeehocZX
++ mktemp
+ local LAST_ERR=/tmp/tmp.vD5WAWoZVu
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.GEbeehocZX
namespace/pxc-operator created
+ cat /tmp/tmp.vD5WAWoZVu
+ rm /tmp/tmp.GEbeehocZX /tmp/tmp.vD5WAWoZVu
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aVPDvde6eW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8VMCZsAX9i
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.aVPDvde6eW
++ cat /tmp/tmp.8VMCZsAX9i
++ rm /tmp/tmp.aVPDvde6eW /tmp/tmp.8VMCZsAX9i
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1588-5c1f635c-6-cluster7 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.VQPpYSDZqs
++ mktemp
+ local LAST_ERR=/tmp/tmp.98c4dRZ2Je
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1588-5c1f635c-6-cluster7 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.VQPpYSDZqs
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1588-5c1f635c-6-cluster7" modified.
+ cat /tmp/tmp.98c4dRZ2Je
+ rm /tmp/tmp.VQPpYSDZqs /tmp/tmp.98c4dRZ2Je
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.jeUE87PJWA
++ mktemp
+ local LAST_ERR=/tmp/tmp.t6g7rHl3jC
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.jeUE87PJWA
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.t6g7rHl3jC
+ rm /tmp/tmp.jeUE87PJWA /tmp/tmp.t6g7rHl3jC
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.12HFbqSxcP
++ mktemp
+ local LAST_ERR=/tmp/tmp.TxyjtZv527
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.12HFbqSxcP
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.TxyjtZv527
+ rm /tmp/tmp.12HFbqSxcP /tmp/tmp.TxyjtZv527
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1588-5c1f635c^'
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/deploy/cw-operator.yaml
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' -
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.8rKKHsmM9e
++ mktemp
+ local LAST_ERR=/tmp/tmp.U8oHAoFOPD
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8rKKHsmM9e
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.U8oHAoFOPD
+ rm /tmp/tmp.8rKKHsmM9e /tmp/tmp.U8oHAoFOPD
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.EAOpM70ele
++ mktemp
+ local LAST_ERR=/tmp/tmp.OZUiV9JMHD
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EAOpM70ele
pod/percona-xtradb-cluster-operator-66688f478-tqfbp condition met
+ cat /tmp/tmp.OZUiV9JMHD
+ rm /tmp/tmp.EAOpM70ele /tmp/tmp.OZUiV9JMHD
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N0Iep4MEUV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nMZHd8ZwWH
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.N0Iep4MEUV
++ cat /tmp/tmp.nMZHd8ZwWH
++ rm /tmp/tmp.N0Iep4MEUV /tmp/tmp.nMZHd8ZwWH
++ return 0
+ wait_pod percona-xtradb-cluster-operator-66688f478-tqfbp 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-66688f478-tqfbp
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-66688f478-tqfbp
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-66688f478-tqfbp condition met
percona-xtradb-cluster-operator-66688f478-tqfbp.Ok
+ sleep 3
+ create_namespace users-29635
+ local namespace=users-29635
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces users-29635'
++ mktemp
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces users-29635
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace users-29635
+ xargs kubectl delete ns
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ local LAST_OUT=/tmp/tmp.tzkh14q5kQ
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.CMLVcka4v7
++ mktemp
+ local LAST_ERR=/tmp/tmp.RLY3jPNfAf
+ local exit_status=0
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.8yrsXBGdSs
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-29635
+ exit_status=0
+ set -e
+ exit_status=1
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ '[' 1 '!=' 0 ']'
+ cat /tmp/tmp.tzkh14q5kQ
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.RLY3jPNfAf
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-29635
+ rm /tmp/tmp.tzkh14q5kQ /tmp/tmp.RLY3jPNfAf
+ return 0
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-29635
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.CMLVcka4v7
+ cat /tmp/tmp.8yrsXBGdSs
Error from server (NotFound): namespaces "users-29635" not found
+ rm /tmp/tmp.CMLVcka4v7 /tmp/tmp.8yrsXBGdSs
+ return 1
+ :
+ wait_for_delete namespace/users-29635
+ local res=namespace/users-29635
+ echo -n 'namespace/users-29635 - '
namespace/users-29635 - + set +o xtrace
Error from server (NotFound): namespaces "users-29635" not found
+ desc 'create namespace users-29635'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace users-29635
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace users-29635
++ mktemp
+ local LAST_OUT=/tmp/tmp.ePJFZBW3bC
++ mktemp
+ local LAST_ERR=/tmp/tmp.6T8K8G6mN8
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace users-29635
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ePJFZBW3bC
namespace/users-29635 created
+ cat /tmp/tmp.6T8K8G6mN8
+ rm /tmp/tmp.ePJFZBW3bC /tmp/tmp.6T8K8G6mN8
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W70YMLhWMl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3eQjRU1I3u
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.W70YMLhWMl
++ cat /tmp/tmp.3eQjRU1I3u
++ rm /tmp/tmp.W70YMLhWMl /tmp/tmp.3eQjRU1I3u
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1588-5c1f635c-6-cluster7 --namespace=users-29635
++ mktemp
+ local LAST_OUT=/tmp/tmp.IkCflTqKJu
++ mktemp
+ local LAST_ERR=/tmp/tmp.re20y2OFso
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1588-5c1f635c-6-cluster7 --namespace=users-29635
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.IkCflTqKJu
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1588-5c1f635c-6-cluster7" modified.
+ cat /tmp/tmp.re20y2OFso
+ rm /tmp/tmp.IkCflTqKJu /tmp/tmp.re20y2OFso
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.PrMLRz4C3b
++ mktemp
+ local LAST_ERR=/tmp/tmp.8DaGJEILn1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.PrMLRz4C3b
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.8DaGJEILn1
+ rm /tmp/tmp.PrMLRz4C3b /tmp/tmp.8DaGJEILn1
+ return 0
+ desc 'create PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC cluster
-----------------------------------------------------------------------------------

+ newpass=test-password
++ echo -n test-password
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZA==
+ cluster=some-name
+ spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/some-name.yml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/some-name.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.sKT7GyVYzE
++ mktemp
+ local LAST_ERR=/tmp/tmp.8sT1bzcGSR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.sKT7GyVYzE
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.8sT1bzcGSR
+ rm /tmp/tmp.sKT7GyVYzE /tmp/tmp.8sT1bzcGSR
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1588-5c1f635c#'
+ local LAST_OUT=/tmp/tmp.O5Ki8P95LI
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-29635~
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
++ mktemp
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ local LAST_ERR=/tmp/tmp.YaU4LS7Ao2
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.O5Ki8P95LI
deployment.apps/pxc-client created
+ cat /tmp/tmp.YaU4LS7Ao2
+ rm /tmp/tmp.O5Ki8P95LI /tmp/tmp.YaU4LS7Ao2
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/some-name.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/some-name.yml
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/some-name.yml
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ local LAST_OUT=/tmp/tmp.nfljAR3yxq
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1588-5c1f635c#'
+ local LAST_ERR=/tmp/tmp.on0kNr1IGS
+ local exit_status=0
++ seq 0 2
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-29635~
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nfljAR3yxq
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.on0kNr1IGS
+ rm /tmp/tmp.nfljAR3yxq /tmp/tmp.on0kNr1IGS
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy some-name
++ local target_cluster=some-name
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.rQv4r6IJXH
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.m4bbUDkwsk
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.rQv4r6IJXH
+++ cat /tmp/tmp.m4bbUDkwsk
+++ rm /tmp/tmp.rQv4r6IJXH /tmp/tmp.m4bbUDkwsk
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.gD5HdsC4fC
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.QIOQOeJCzM
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.gD5HdsC4fC
+++ cat /tmp/tmp.QIOQOeJCzM
+++ rm /tmp/tmp.gD5HdsC4fC /tmp/tmp.QIOQOeJCzM
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo some-name-proxysql
++ return
+ local proxy=some-name-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-29635
++ mktemp
+ local LAST_OUT=/tmp/tmp.mR2C9UHH0j
++ mktemp
+ local LAST_ERR=/tmp/tmp.dzcUhcntY0
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-29635
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-29635
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-29635
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.mR2C9UHH0j
+ cat /tmp/tmp.dzcUhcntY0
error: no matching resources found
+ rm /tmp/tmp.mR2C9UHH0j /tmp/tmp.dzcUhcntY0
+ return 1
+ true
+ wait_for_running some-name-proxysql 1
+ local name=some-name-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-proxysql-0 480
+ local pod=some-name-proxysql-0
+ local max_retry=480
+ local ns=
++ echo some-name-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
pod/some-name-proxysql-0 condition met
some-name-proxysql-0.Ok
+ wait_for_running some-name-pxc 3
+ local name=some-name-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
some-name-pxc-0.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-1 480
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
some-name-pxc-1.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-2 480
+ local pod=some-name-pxc-2
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-2 condition met
some-name-pxc-2.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-proxysql -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h some-name-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XzDFwvX9fV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BNDonRGn2A
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XzDFwvX9fV
++ cat /tmp/tmp.BNDonRGn2A
++ rm /tmp/tmp.XzDFwvX9fV /tmp/tmp.BNDonRGn2A
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-proxysql -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h some-name-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.atsWlPxqYk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cxJYNbHwsq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.atsWlPxqYk
++ cat /tmp/tmp.cxJYNbHwsq
++ rm /tmp/tmp.atsWlPxqYk /tmp/tmp.cxJYNbHwsq
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vAHO9wfWyX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QeRBym3Gyl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vAHO9wfWyX
++ cat /tmp/tmp.QeRBym3Gyl
++ rm /tmp/tmp.vAHO9wfWyX /tmp/tmp.QeRBym3Gyl
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1.sql /tmp/tmp.T7soxWEpSM/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Icb3L1sSZb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.64nksv777J
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Icb3L1sSZb
++ cat /tmp/tmp.64nksv777J
++ rm /tmp/tmp.Icb3L1sSZb /tmp/tmp.64nksv777J
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1.sql /tmp/tmp.T7soxWEpSM/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZYi0CHP4iI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KWGn9bLKmA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZYi0CHP4iI
++ cat /tmp/tmp.KWGn9bLKmA
++ rm /tmp/tmp.ZYi0CHP4iI /tmp/tmp.KWGn9bLKmA
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1.sql /tmp/tmp.T7soxWEpSM/select-1.sql
++ is_keyring_plugin_in_use some-name
++ local cluster=some-name
++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d12wWcaEFS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mmwqFs31YX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.d12wWcaEFS
++ cat /tmp/tmp.mmwqFs31YX
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.d12wWcaEFS /tmp/tmp.mmwqFs31YX
++ return 0
+ '[' '' ']'
+ desc 'test root'
+ set +o xtrace


-----------------------------------------------------------------------------------
test root
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets root dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=root
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.o1mPrJoU3b
++ mktemp
+ local LAST_ERR=/tmp/tmp.s8zHLgGXBX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.o1mPrJoU3b
secret/my-cluster-secrets patched
+ cat /tmp/tmp.s8zHLgGXBX
+ rm /tmp/tmp.o1mPrJoU3b /tmp/tmp.s8zHLgGXBX
+ return 0
+ sleep 15
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Bl7UUD6N6P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TEzSPQGlTj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Bl7UUD6N6P
++ cat /tmp/tmp.TEzSPQGlTj
++ rm /tmp/tmp.Bl7UUD6N6P /tmp/tmp.TEzSPQGlTj
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.T7soxWEpSM/select-4.sql
+ desc 'test proxyadmin'
+ set +o xtrace


-----------------------------------------------------------------------------------
test proxyadmin
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.owAHQc7qWK
++ mktemp
+ local LAST_ERR=/tmp/tmp.Q3XFQBdF7U
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.owAHQc7qWK
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.Q3XFQBdF7U
+ rm /tmp/tmp.owAHQc7qWK /tmp/tmp.Q3XFQBdF7U
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bwvcsGWJ0i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ii9wZwPQ2j
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bwvcsGWJ0i
++ cat /tmp/tmp.ii9wZwPQ2j
++ rm /tmp/tmp.bwvcsGWJ0i /tmp/tmp.ii9wZwPQ2j
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L7lweCC8z7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.I5L0YiAJNI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.L7lweCC8z7
++ cat /tmp/tmp.I5L0YiAJNI
++ rm /tmp/tmp.L7lweCC8z7 /tmp/tmp.I5L0YiAJNI
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.SIGQk6o02K
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.0lcAPi2kXD
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.SIGQk6o02K
+++++ cat /tmp/tmp.0lcAPi2kXD
+++++ rm /tmp/tmp.SIGQk6o02K /tmp/tmp.0lcAPi2kXD
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.n3rpfmIRib
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.ansAXk6vKF
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.n3rpfmIRib
+++++ cat /tmp/tmp.ansAXk6vKF
+++++ rm /tmp/tmp.n3rpfmIRib /tmp/tmp.ansAXk6vKF
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5XucUKbBP7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l3uJcgHCvK
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.5XucUKbBP7
++ cat /tmp/tmp.l3uJcgHCvK
++ rm /tmp/tmp.5XucUKbBP7 /tmp/tmp.l3uJcgHCvK
++ return 0
+ [[ 3 == \3 ]]
+ patch_secret my-cluster-secrets proxyadmin dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=proxyadmin
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.7WPcWDAJHY
++ mktemp
+ local LAST_ERR=/tmp/tmp.PVUnabuQ53
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7WPcWDAJHY
secret/my-cluster-secrets patched
+ cat /tmp/tmp.PVUnabuQ53
+ rm /tmp/tmp.7WPcWDAJHY /tmp/tmp.PVUnabuQ53
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.USNyut2TvY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DLJbaPCdqi
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.USNyut2TvY
++ cat /tmp/tmp.DLJbaPCdqi
++ rm /tmp/tmp.USNyut2TvY /tmp/tmp.DLJbaPCdqi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ioC9WT5lHc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.68FT785elW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ioC9WT5lHc
++ cat /tmp/tmp.68FT785elW
++ rm /tmp/tmp.ioC9WT5lHc /tmp/tmp.68FT785elW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TnAvTbRRKP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HUayzwBPwA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TnAvTbRRKP
++ cat /tmp/tmp.HUayzwBPwA
++ rm /tmp/tmp.TnAvTbRRKP /tmp/tmp.HUayzwBPwA
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZLlvwWT5qx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3xbEOrZeM8
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZLlvwWT5qx
++ cat /tmp/tmp.3xbEOrZeM8
++ rm /tmp/tmp.ZLlvwWT5qx /tmp/tmp.3xbEOrZeM8
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.rKencheT4N
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.XghhyWazZK
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.rKencheT4N
+++++ cat /tmp/tmp.XghhyWazZK
+++++ rm /tmp/tmp.rKencheT4N /tmp/tmp.XghhyWazZK
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.d7416bpt52
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.eWYkJiyPi8
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.d7416bpt52
+++++ cat /tmp/tmp.eWYkJiyPi8
+++++ rm /tmp/tmp.d7416bpt52 /tmp/tmp.eWYkJiyPi8
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LcWSAAS7tX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0wTHUym6mq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LcWSAAS7tX
++ cat /tmp/tmp.0wTHUym6mq
++ rm /tmp/tmp.LcWSAAS7tX /tmp/tmp.0wTHUym6mq
++ return 0
+ [[ 3 == \3 ]]
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-0
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2-80.sql ']'
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-0
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2.sql /tmp/tmp.T7soxWEpSM/select-2.sql
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-1
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2-80.sql ']'
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-1
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2.sql /tmp/tmp.T7soxWEpSM/select-2.sql
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-2
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2-80.sql ']'
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-2
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2.sql /tmp/tmp.T7soxWEpSM/select-2.sql
+ desc 'test xtrabackup'
+ set +o xtrace


-----------------------------------------------------------------------------------
test xtrabackup
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.CtAstt2yam
++ mktemp
+ local LAST_ERR=/tmp/tmp.iYkpYhUdGc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.CtAstt2yam
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.iYkpYhUdGc
+ rm /tmp/tmp.CtAstt2yam /tmp/tmp.iYkpYhUdGc
+ return 0
+ patch_secret my-cluster-secrets xtrabackup dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=xtrabackup
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.h2tnDdEx7X
++ mktemp
+ local LAST_ERR=/tmp/tmp.IZQnqbAtGz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.h2tnDdEx7X
secret/my-cluster-secrets patched
+ cat /tmp/tmp.IZQnqbAtGz
+ rm /tmp/tmp.h2tnDdEx7X /tmp/tmp.IZQnqbAtGz
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EcaFcPw7Tl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nZIcz031hL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.EcaFcPw7Tl
++ cat /tmp/tmp.nZIcz031hL
++ rm /tmp/tmp.EcaFcPw7Tl /tmp/tmp.nZIcz031hL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uhJIQAKfLw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v8asPOHFYT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.uhJIQAKfLw
++ cat /tmp/tmp.v8asPOHFYT
++ rm /tmp/tmp.uhJIQAKfLw /tmp/tmp.v8asPOHFYT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nQ7bSksvGO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mxVzJP3v67
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nQ7bSksvGO
++ cat /tmp/tmp.mxVzJP3v67
++ rm /tmp/tmp.nQ7bSksvGO /tmp/tmp.mxVzJP3v67
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.v4CQczEIah
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qG4gXtiGqx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.v4CQczEIah
++ cat /tmp/tmp.qG4gXtiGqx
++ rm /tmp/tmp.v4CQczEIah /tmp/tmp.qG4gXtiGqx
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MUsMuy6t2i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yvqGQcTArk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.MUsMuy6t2i
++ cat /tmp/tmp.yvqGQcTArk
++ rm /tmp/tmp.MUsMuy6t2i /tmp/tmp.yvqGQcTArk
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PAgS0gRIpg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yQQj8A2Adm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PAgS0gRIpg
++ cat /tmp/tmp.yQQj8A2Adm
++ rm /tmp/tmp.PAgS0gRIpg /tmp/tmp.yQQj8A2Adm
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H3NC11GcHY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UtXDEdypcJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.H3NC11GcHY
++ cat /tmp/tmp.UtXDEdypcJ
++ rm /tmp/tmp.H3NC11GcHY /tmp/tmp.UtXDEdypcJ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XVY0RSnlFU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1zypFd491p
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XVY0RSnlFU
++ cat /tmp/tmp.1zypFd491p
++ rm /tmp/tmp.XVY0RSnlFU /tmp/tmp.1zypFd491p
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UGhr7tup7u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fjRiQ8SPTY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.UGhr7tup7u
++ cat /tmp/tmp.fjRiQ8SPTY
++ rm /tmp/tmp.UGhr7tup7u /tmp/tmp.fjRiQ8SPTY
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.EEPT9fpRSe
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.uK9WjEhANV
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.EEPT9fpRSe
+++++ cat /tmp/tmp.uK9WjEhANV
+++++ rm /tmp/tmp.EEPT9fpRSe /tmp/tmp.uK9WjEhANV
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.DNT5gE13cZ
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.Bq68X6o7Mz
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.DNT5gE13cZ
+++++ cat /tmp/tmp.Bq68X6o7Mz
+++++ rm /tmp/tmp.DNT5gE13cZ /tmp/tmp.Bq68X6o7Mz
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QnAgFcQijV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YQoEVb1JH3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.QnAgFcQijV
++ cat /tmp/tmp.YQoEVb1JH3
++ rm /tmp/tmp.QnAgFcQijV /tmp/tmp.YQoEVb1JH3
++ return 0
+ [[ 2 == \2 ]]
+ compare_mysql_cmd_local select-3 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 '' pxc
+ local command_id=select-3
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\'''
+ local pod=some-name-pxc-0
+ local postfix=
+ local container_name=pxc
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-3-80.sql ']'
+ run_mysql_local 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 pxc
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\'''
+ local pod=some-name-pxc-0
+ local container_name=pxc
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-3.sql /tmp/tmp.T7soxWEpSM/select-3.sql
+ desc 'test monitor'
+ set +o xtrace


-----------------------------------------------------------------------------------
test monitor
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=monitor
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.PSvQgVbdPw
++ mktemp
+ local LAST_ERR=/tmp/tmp.N0jL8eMmkv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.PSvQgVbdPw
secret/my-cluster-secrets patched
+ cat /tmp/tmp.N0jL8eMmkv
+ rm /tmp/tmp.PSvQgVbdPw /tmp/tmp.N0jL8eMmkv
+ return 0
+ wait_for_password_propagation my-cluster-secrets monitor
+ local secret=my-cluster-secrets
+ local user=monitor
+ local max_retry=240
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.M9mbD56BGP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AhaxFBJb60
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.M9mbD56BGP
++ cat /tmp/tmp.AhaxFBJb60
++ rm /tmp/tmp.M9mbD56BGP /tmp/tmp.AhaxFBJb60
++ return 0
+ local root_pass=test-password
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ retry=0
+ is_password_updated monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep additional_password
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sqlsdOXGJA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BxejYsDZ8b
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sqlsdOXGJA
++ cat /tmp/tmp.BxejYsDZ8b
++ rm /tmp/tmp.sqlsdOXGJA /tmp/tmp.BxejYsDZ8b
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
{"additional_password": "*C1F414D9BAF378B656A849B31F9F8AF3125F558B"}
+ retry=0
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4mfC8p5OkI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lSFN9mBdyc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4mfC8p5OkI
++ cat /tmp/tmp.lSFN9mBdyc
++ rm /tmp/tmp.4mfC8p5OkI /tmp/tmp.lSFN9mBdyc
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 1 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ grep NULL
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oVrhqiBaK3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dWDwuHyHIz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oVrhqiBaK3
++ cat /tmp/tmp.dWDwuHyHIz
++ rm /tmp/tmp.oVrhqiBaK3 /tmp/tmp.dWDwuHyHIz
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 2 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mYOb1ougwO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tRuGaGZlJp
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.mYOb1ougwO
++ cat /tmp/tmp.tRuGaGZlJp
++ rm /tmp/tmp.mYOb1ougwO /tmp/tmp.tRuGaGZlJp
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 3 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ grep NULL
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7RTQXy2u0Z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AVKoHlX2Z8
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7RTQXy2u0Z
++ cat /tmp/tmp.AVKoHlX2Z8
++ rm /tmp/tmp.7RTQXy2u0Z /tmp/tmp.AVKoHlX2Z8
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 4 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MnRMK52TcL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2GvSHIANv2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.MnRMK52TcL
++ cat /tmp/tmp.2GvSHIANv2
++ rm /tmp/tmp.MnRMK52TcL /tmp/tmp.2GvSHIANv2
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 5 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q5dWvaV6mm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZPKiA4195z
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.q5dWvaV6mm
++ cat /tmp/tmp.ZPKiA4195z
++ rm /tmp/tmp.q5dWvaV6mm /tmp/tmp.ZPKiA4195z
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 6 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ grep NULL
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y6J0iOyUKT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UUKN5UQXLd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.y6J0iOyUKT
++ cat /tmp/tmp.UUKN5UQXLd
++ rm /tmp/tmp.y6J0iOyUKT /tmp/tmp.UUKN5UQXLd
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 7 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C6EKHorCWk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ospNSH9awe
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.C6EKHorCWk
++ cat /tmp/tmp.ospNSH9awe
++ rm /tmp/tmp.C6EKHorCWk /tmp/tmp.ospNSH9awe
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 8 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.r09BqRk29X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0UhEPATzS9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.r09BqRk29X
++ cat /tmp/tmp.0UhEPATzS9
++ rm /tmp/tmp.r09BqRk29X /tmp/tmp.0UhEPATzS9
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 9 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Fv8LehIJdW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6Wckj2laGI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Fv8LehIJdW
++ cat /tmp/tmp.6Wckj2laGI
++ rm /tmp/tmp.Fv8LehIJdW /tmp/tmp.6Wckj2laGI
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 10 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0RJoX2h6jW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LIanljs2bg
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0RJoX2h6jW
++ cat /tmp/tmp.LIanljs2bg
++ rm /tmp/tmp.0RJoX2h6jW /tmp/tmp.LIanljs2bg
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
NULL
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AUjosawYJ4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vb7EbEG0u5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AUjosawYJ4
++ cat /tmp/tmp.Vb7EbEG0u5
++ rm /tmp/tmp.AUjosawYJ4 /tmp/tmp.Vb7EbEG0u5
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BKeVXQBh1m
+++ mktemp
++ local LAST_ERR=/tmp/tmp.naLH3B9Req
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BKeVXQBh1m
++ cat /tmp/tmp.naLH3B9Req
++ rm /tmp/tmp.BKeVXQBh1m /tmp/tmp.naLH3B9Req
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.UWSZttWI6B
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.2UCUO6ONif
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.UWSZttWI6B
+++++ cat /tmp/tmp.2UCUO6ONif
+++++ rm /tmp/tmp.UWSZttWI6B /tmp/tmp.2UCUO6ONif
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.F6SFnoGDcy
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.fUAm46q4Pv
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.F6SFnoGDcy
+++++ cat /tmp/tmp.fUAm46q4Pv
+++++ rm /tmp/tmp.F6SFnoGDcy /tmp/tmp.fUAm46q4Pv
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XncdFVRfa9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kTvfm4bkkh
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XncdFVRfa9
++ cat /tmp/tmp.kTvfm4bkkh
++ rm /tmp/tmp.XncdFVRfa9 /tmp/tmp.kTvfm4bkkh
++ return 0
+ [[ 2 == \2 ]]
+ sleep 10
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gJiyB5VJEg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8NBJQIIXqP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gJiyB5VJEg
++ cat /tmp/tmp.8NBJQIIXqP
++ rm /tmp/tmp.gJiyB5VJEg /tmp/tmp.8NBJQIIXqP
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.T7soxWEpSM/select-4.sql
+ desc 'test operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
test operator
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets operator dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=operator
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.kJ3VJbNMiT
++ mktemp
+ local LAST_ERR=/tmp/tmp.rB38bh63Oo
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kJ3VJbNMiT
secret/my-cluster-secrets patched
+ cat /tmp/tmp.rB38bh63Oo
+ rm /tmp/tmp.kJ3VJbNMiT /tmp/tmp.rB38bh63Oo
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3A58aF0gS6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wAHOU7G8pN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3A58aF0gS6
++ cat /tmp/tmp.wAHOU7G8pN
++ rm /tmp/tmp.3A58aF0gS6 /tmp/tmp.wAHOU7G8pN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IxDpgT9f1N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WvylUFHb5z
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IxDpgT9f1N
++ cat /tmp/tmp.WvylUFHb5z
++ rm /tmp/tmp.IxDpgT9f1N /tmp/tmp.WvylUFHb5z
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XT3lMW0PSX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0NhepvgRnW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XT3lMW0PSX
++ cat /tmp/tmp.0NhepvgRnW
++ rm /tmp/tmp.XT3lMW0PSX /tmp/tmp.0NhepvgRnW
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.IzRIYv8gAN
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.wfuGdeFbhD
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.IzRIYv8gAN
+++++ cat /tmp/tmp.wfuGdeFbhD
+++++ rm /tmp/tmp.IzRIYv8gAN /tmp/tmp.wfuGdeFbhD
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.L2vXRWXlMq
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.Zx4SfJtimJ
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.L2vXRWXlMq
+++++ cat /tmp/tmp.Zx4SfJtimJ
+++++ rm /tmp/tmp.L2vXRWXlMq /tmp/tmp.Zx4SfJtimJ
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9vPNrgdKgf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ilPQGQEfRT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9vPNrgdKgf
++ cat /tmp/tmp.ilPQGQEfRT
++ rm /tmp/tmp.9vPNrgdKgf /tmp/tmp.ilPQGQEfRT
++ return 0
+ [[ 2 == \2 ]]
+ sleep 10
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nBs4Qoy0Vd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K8tQbKgAgc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nBs4Qoy0Vd
++ cat /tmp/tmp.K8tQbKgAgc
++ rm /tmp/tmp.nBs4Qoy0Vd /tmp/tmp.K8tQbKgAgc
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.T7soxWEpSM/select-4.sql
+ desc 'change secret name'
+ set +o xtrace


-----------------------------------------------------------------------------------
change secret name
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.S56VPiM6Wi
++ mktemp
+ local LAST_ERR=/tmp/tmp.fql5GxG5L3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.S56VPiM6Wi
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.fql5GxG5L3
+ rm /tmp/tmp.S56VPiM6Wi /tmp/tmp.fql5GxG5L3
+ return 0
+ sleep 30
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P22BN0XbgW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UhENm2S2R1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.P22BN0XbgW
++ cat /tmp/tmp.UhENm2S2R1
++ rm /tmp/tmp.P22BN0XbgW /tmp/tmp.UhENm2S2R1
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.25Ck4qxDYI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8MSWVb1yWf
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.25Ck4qxDYI
++ cat /tmp/tmp.8MSWVb1yWf
++ rm /tmp/tmp.25Ck4qxDYI /tmp/tmp.8MSWVb1yWf
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AMOBVpykMR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wBaBrr672o
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AMOBVpykMR
++ cat /tmp/tmp.wBaBrr672o
++ rm /tmp/tmp.AMOBVpykMR /tmp/tmp.wBaBrr672o
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cdKAeULFy4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CsekikXhsQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cdKAeULFy4
++ cat /tmp/tmp.CsekikXhsQ
++ rm /tmp/tmp.cdKAeULFy4 /tmp/tmp.CsekikXhsQ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iqXkgBAgC6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N5vG73v2L6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.iqXkgBAgC6
++ cat /tmp/tmp.N5vG73v2L6
++ rm /tmp/tmp.iqXkgBAgC6 /tmp/tmp.N5vG73v2L6
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TEKWgVuFLS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0JTCFicIl8
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TEKWgVuFLS
++ cat /tmp/tmp.0JTCFicIl8
++ rm /tmp/tmp.TEKWgVuFLS /tmp/tmp.0JTCFicIl8
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YueIJZ9V3Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g4JJ2SvYLU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YueIJZ9V3Q
++ cat /tmp/tmp.g4JJ2SvYLU
++ rm /tmp/tmp.YueIJZ9V3Q /tmp/tmp.g4JJ2SvYLU
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7WnWXD0C5K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xlWWYWw4hj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7WnWXD0C5K
++ cat /tmp/tmp.xlWWYWw4hj
++ rm /tmp/tmp.7WnWXD0C5K /tmp/tmp.xlWWYWw4hj
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.7KnwE0dwzQ
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.vx8IXxM56E
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.7KnwE0dwzQ
+++++ cat /tmp/tmp.vx8IXxM56E
+++++ rm /tmp/tmp.7KnwE0dwzQ /tmp/tmp.vx8IXxM56E
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.CUtITnLV3A
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.9eOT9hsrzG
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.CUtITnLV3A
+++++ cat /tmp/tmp.9eOT9hsrzG
+++++ rm /tmp/tmp.CUtITnLV3A /tmp/tmp.9eOT9hsrzG
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jzmshViYyh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3nKETAN5Vl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.jzmshViYyh
++ cat /tmp/tmp.3nKETAN5Vl
++ rm /tmp/tmp.jzmshViYyh /tmp/tmp.3nKETAN5Vl
++ return 0
+ [[ 2 == \2 ]]
+ desc 'test new operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
test new operator
-----------------------------------------------------------------------------------

+ newpass=test-password2
++ echo -n test-password2
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZDI=
+ patch_secret my-cluster-secrets-2 operator dGVzdC1wYXNzd29yZDI=
+ local secret=my-cluster-secrets-2
+ local key=operator
+ local value=dGVzdC1wYXNzd29yZDI=
+ kubectl_bin patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.QxcdDwmWE9
++ mktemp
+ local LAST_ERR=/tmp/tmp.N8SwEXUumm
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QxcdDwmWE9
secret/my-cluster-secrets-2 patched
+ cat /tmp/tmp.N8SwEXUumm
+ rm /tmp/tmp.QxcdDwmWE9 /tmp/tmp.N8SwEXUumm
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kNzV737okO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.220PUMXbA2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kNzV737okO
++ cat /tmp/tmp.220PUMXbA2
++ rm /tmp/tmp.kNzV737okO /tmp/tmp.220PUMXbA2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1B4OQCblEf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QIxl0CNvrY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1B4OQCblEf
++ cat /tmp/tmp.QIxl0CNvrY
++ rm /tmp/tmp.1B4OQCblEf /tmp/tmp.QIxl0CNvrY
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ITzwEkfA0a
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xgcVuuiGoC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ITzwEkfA0a
++ cat /tmp/tmp.xgcVuuiGoC
++ rm /tmp/tmp.ITzwEkfA0a /tmp/tmp.xgcVuuiGoC
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.nAbS3Gwct9
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.GT2PtwsStH
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.nAbS3Gwct9
+++++ cat /tmp/tmp.GT2PtwsStH
+++++ rm /tmp/tmp.nAbS3Gwct9 /tmp/tmp.GT2PtwsStH
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.qnTtB1otPF
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.0v69jFULbo
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.qnTtB1otPF
+++++ cat /tmp/tmp.0v69jFULbo
+++++ rm /tmp/tmp.qnTtB1otPF /tmp/tmp.0v69jFULbo
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YHF81EqtmU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uUQH49RiZM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YHF81EqtmU
++ cat /tmp/tmp.uUQH49RiZM
++ rm /tmp/tmp.YHF81EqtmU /tmp/tmp.uUQH49RiZM
++ return 0
+ [[ 2 == \2 ]]
+ sleep 20
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0Ipyf0qtbT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sIIgMX0o9q
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0Ipyf0qtbT
++ cat /tmp/tmp.sIIgMX0o9q
++ rm /tmp/tmp.0Ipyf0qtbT /tmp/tmp.sIIgMX0o9q
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.T7soxWEpSM/select-4.sql
++ getSecretData my-cluster-secrets-2 root
++ local secretName=my-cluster-secrets-2
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets-2 '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0ygwU9oaek
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3dGy0qtId3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets-2 '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0ygwU9oaek
++ cat /tmp/tmp.3dGy0qtId3
++ rm /tmp/tmp.0ygwU9oaek /tmp/tmp.3dGy0qtId3
++ return 0
+ newpass=dN%@pSkTIu9#pQ~mp
+ desc 'test new users sync'
+ set +o xtrace


-----------------------------------------------------------------------------------
test new users sync
-----------------------------------------------------------------------------------

+ run_mysql 'CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\''dN%@pSkTIu9#pQ~mp'\'';' '-h some-name-pxc -uroot -p'\''dN%@pSkTIu9#pQ~mp'\'''
+ local 'command=CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\''dN%@pSkTIu9#pQ~mp'\'';'
+ local 'uri=-h some-name-pxc -uroot -p'\''dN%@pSkTIu9#pQ~mp'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rmMX0pPpSJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Xp3rGNMsaV
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rmMX0pPpSJ
++ cat /tmp/tmp.Xp3rGNMsaV
++ rm /tmp/tmp.rmMX0pPpSJ /tmp/tmp.Xp3rGNMsaV
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ sleep 40
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\''dN%@pSkTIu9#pQ~mp'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -utestsync -p'\''dN%@pSkTIu9#pQ~mp'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\''dN%@pSkTIu9#pQ~mp'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -utestsync -p'\''dN%@pSkTIu9#pQ~mp'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6YMdOGW3TV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X18nMN5r5W
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6YMdOGW3TV
++ cat /tmp/tmp.X18nMN5r5W
++ rm /tmp/tmp.6YMdOGW3TV /tmp/tmp.X18nMN5r5W
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.T7soxWEpSM/select-4.sql
++ getSecretData internal-some-name operator
++ local secretName=internal-some-name
++ local dataKey=operator
++ kubectl_bin get secrets/internal-some-name '--template={{.data.operator}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N1PbiPhz89
+++ mktemp
++ local LAST_ERR=/tmp/tmp.REMkSqA4DE
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/internal-some-name '--template={{.data.operator}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.N1PbiPhz89
++ cat /tmp/tmp.REMkSqA4DE
++ rm /tmp/tmp.N1PbiPhz89 /tmp/tmp.REMkSqA4DE
++ return 0
+ pass=test-password2
+ desc 'check secret without operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
check secret without operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.67WwbnPZfe
++ mktemp
+ local LAST_ERR=/tmp/tmp.YoTRqgdJX8
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.67WwbnPZfe
secret/my-cluster-secrets-2 configured
+ cat /tmp/tmp.YoTRqgdJX8
Warning: resource secrets/my-cluster-secrets-2 is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.67WwbnPZfe /tmp/tmp.YoTRqgdJX8
+ return 0
+ sleep 15
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PS4Qes66Hc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.naby2d1J04
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PS4Qes66Hc
++ cat /tmp/tmp.naby2d1J04
++ rm /tmp/tmp.PS4Qes66Hc /tmp/tmp.naby2d1J04
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.T7soxWEpSM/select-4.sql
+ newpass=test-password2
++ echo -n test-password2
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZDI=
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/conf/some-name.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1588-5c1f635c#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
++ mktemp
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-29635~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ local LAST_OUT=/tmp/tmp.vRTb6PCBlq
++ mktemp
+ local LAST_ERR=/tmp/tmp.6ZXkMdaZrx
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vRTb6PCBlq
perconaxtradbcluster.pxc.percona.com/some-name configured
+ cat /tmp/tmp.6ZXkMdaZrx
+ rm /tmp/tmp.vRTb6PCBlq /tmp/tmp.6ZXkMdaZrx
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bAE7m5oOOZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.auEpKgQOwB
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bAE7m5oOOZ
++ cat /tmp/tmp.auEpKgQOwB
++ rm /tmp/tmp.bAE7m5oOOZ /tmp/tmp.auEpKgQOwB
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Et3hRTUl0r
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W2VXxGEuw5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Et3hRTUl0r
++ cat /tmp/tmp.W2VXxGEuw5
++ rm /tmp/tmp.Et3hRTUl0r /tmp/tmp.W2VXxGEuw5
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eYJh3Gyu2Z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XQyoWpNkqz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eYJh3Gyu2Z
++ cat /tmp/tmp.XQyoWpNkqz
++ rm /tmp/tmp.eYJh3Gyu2Z /tmp/tmp.XQyoWpNkqz
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RF93hdgiST
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aHuuM1DqKQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RF93hdgiST
++ cat /tmp/tmp.aHuuM1DqKQ
++ rm /tmp/tmp.RF93hdgiST /tmp/tmp.aHuuM1DqKQ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.78krj4rKfx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LNaHZykOFZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.78krj4rKfx
++ cat /tmp/tmp.LNaHZykOFZ
++ rm /tmp/tmp.78krj4rKfx /tmp/tmp.LNaHZykOFZ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F7cwjLSoqb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z0AapKwqTM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.F7cwjLSoqb
++ cat /tmp/tmp.Z0AapKwqTM
++ rm /tmp/tmp.F7cwjLSoqb /tmp/tmp.Z0AapKwqTM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tgJTrkOljN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mZmn39zLYJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tgJTrkOljN
++ cat /tmp/tmp.mZmn39zLYJ
++ rm /tmp/tmp.tgJTrkOljN /tmp/tmp.mZmn39zLYJ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9KBq2U0bqO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5GfxeNtubu
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9KBq2U0bqO
++ cat /tmp/tmp.5GfxeNtubu
++ rm /tmp/tmp.9KBq2U0bqO /tmp/tmp.5GfxeNtubu
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ypbrZg3byC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4hCTPXpIM9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ypbrZg3byC
++ cat /tmp/tmp.4hCTPXpIM9
++ rm /tmp/tmp.ypbrZg3byC /tmp/tmp.4hCTPXpIM9
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hmmLiopjKc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8csYvrgbgM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hmmLiopjKc
++ cat /tmp/tmp.8csYvrgbgM
++ rm /tmp/tmp.hmmLiopjKc /tmp/tmp.8csYvrgbgM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KrB1V96uv9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xxNqbyVQq7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KrB1V96uv9
++ cat /tmp/tmp.xxNqbyVQq7
++ rm /tmp/tmp.KrB1V96uv9 /tmp/tmp.xxNqbyVQq7
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HsgYwQiX0S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nv5HKvp6o0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.HsgYwQiX0S
++ cat /tmp/tmp.nv5HKvp6o0
++ rm /tmp/tmp.HsgYwQiX0S /tmp/tmp.nv5HKvp6o0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ni2sDwVH7Y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vyFoUuJdqj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Ni2sDwVH7Y
++ cat /tmp/tmp.vyFoUuJdqj
++ rm /tmp/tmp.Ni2sDwVH7Y /tmp/tmp.vyFoUuJdqj
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t1hHvYSkn0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qzZdzxfUCu
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.t1hHvYSkn0
++ cat /tmp/tmp.qzZdzxfUCu
++ rm /tmp/tmp.t1hHvYSkn0 /tmp/tmp.qzZdzxfUCu
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.icJs6crKY9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AX73qJpWgl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.icJs6crKY9
++ cat /tmp/tmp.AX73qJpWgl
++ rm /tmp/tmp.icJs6crKY9 /tmp/tmp.AX73qJpWgl
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZxFHvw22Qv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BVpO45Dyws
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZxFHvw22Qv
++ cat /tmp/tmp.BVpO45Dyws
++ rm /tmp/tmp.ZxFHvw22Qv /tmp/tmp.BVpO45Dyws
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mRqq74wjt0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bLUQuNNYjY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.mRqq74wjt0
++ cat /tmp/tmp.bLUQuNNYjY
++ rm /tmp/tmp.mRqq74wjt0 /tmp/tmp.bLUQuNNYjY
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oTGPuzYXcI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wUNaxYPH70
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oTGPuzYXcI
++ cat /tmp/tmp.wUNaxYPH70
++ rm /tmp/tmp.oTGPuzYXcI /tmp/tmp.wUNaxYPH70
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.K8Ra1It5mc
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.duZGirMjl8
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.K8Ra1It5mc
+++++ cat /tmp/tmp.duZGirMjl8
+++++ rm /tmp/tmp.K8Ra1It5mc /tmp/tmp.duZGirMjl8
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YYiA9ih4BH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i9aR8DniY1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YYiA9ih4BH
++ cat /tmp/tmp.i9aR8DniY1
++ rm /tmp/tmp.YYiA9ih4BH /tmp/tmp.i9aR8DniY1
++ return 0
+ [[ 3 == \3 ]]
+ patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZDI=
+ local secret=my-cluster-secrets
+ local key=monitor
+ local value=dGVzdC1wYXNzd29yZDI=
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.qAGLFQpQzn
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZIfwVexqMY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.qAGLFQpQzn
secret/my-cluster-secrets patched
+ cat /tmp/tmp.ZIfwVexqMY
+ rm /tmp/tmp.qAGLFQpQzn /tmp/tmp.ZIfwVexqMY
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XSx99h0THd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DpA2t4qNlk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XSx99h0THd
++ cat /tmp/tmp.DpA2t4qNlk
++ rm /tmp/tmp.XSx99h0THd /tmp/tmp.DpA2t4qNlk
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.URaBMWaZGG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WaXPnGu5Kr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.URaBMWaZGG
++ cat /tmp/tmp.WaXPnGu5Kr
++ rm /tmp/tmp.URaBMWaZGG /tmp/tmp.WaXPnGu5Kr
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Id9rfDqcpz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0jFxHi3hSn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Id9rfDqcpz
++ cat /tmp/tmp.0jFxHi3hSn
++ rm /tmp/tmp.Id9rfDqcpz /tmp/tmp.0jFxHi3hSn
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eU2kkMVFMZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Zzy8lQeGqe
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eU2kkMVFMZ
++ cat /tmp/tmp.Zzy8lQeGqe
++ rm /tmp/tmp.eU2kkMVFMZ /tmp/tmp.Zzy8lQeGqe
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AXFBykq1kD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zHRT5JOYHO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AXFBykq1kD
++ cat /tmp/tmp.zHRT5JOYHO
++ rm /tmp/tmp.AXFBykq1kD /tmp/tmp.zHRT5JOYHO
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.J6Ky2GDPEo
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.fbUuTH9Wur
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.J6Ky2GDPEo
+++++ cat /tmp/tmp.fbUuTH9Wur
+++++ rm /tmp/tmp.J6Ky2GDPEo /tmp/tmp.fbUuTH9Wur
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gR3mJT2O5D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MzCEQgS6FI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gR3mJT2O5D
++ cat /tmp/tmp.MzCEQgS6FI
++ rm /tmp/tmp.gR3mJT2O5D /tmp/tmp.MzCEQgS6FI
++ return 0
+ [[ 3 == \3 ]]
+ compare_mysql_cmd select-3 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local command_id=select-3
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-3-80.sql ']'
+ run_mysql 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1TsvQ423u8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VGHTl0kAQk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1TsvQ423u8
++ cat /tmp/tmp.VGHTl0kAQk
++ rm /tmp/tmp.1TsvQ423u8 /tmp/tmp.VGHTl0kAQk
++ return 0
+ client_pod=pxc-client-65d95667fc-77k84
+ wait_pod pxc-client-65d95667fc-77k84
+ local pod=pxc-client-65d95667fc-77k84
+ local max_retry=480
+ local ns=
++ echo pxc-client-65d95667fc-77k84
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-65d95667fc-77k84 condition met
pxc-client-65d95667fc-77k84.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.T7soxWEpSM/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-3.sql /tmp/tmp.T7soxWEpSM/select-3.sql
+ destroy users-29635
+ local namespace=users-29635
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ grep -v level=info
+ grep -v 'the object has been modified'
+ grep -v 'get backup status: Job.batch'
+ tee /tmp/tmp.T7soxWEpSM/operator.log
+ sort -u
++ get_operator_pod
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nFJZQsRo9n
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3EqnVMRhKi
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nFJZQsRo9n
++ cat /tmp/tmp.3EqnVMRhKi
++ rm /tmp/tmp.nFJZQsRo9n /tmp/tmp.3EqnVMRhKi
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-66688f478-tqfbp
++ mktemp
+ local LAST_OUT=/tmp/tmp.UTeBjzj5LF
++ mktemp
+ local LAST_ERR=/tmp/tmp.0s48wMEdZl
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-66688f478-tqfbp
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UTeBjzj5LF
+ cat /tmp/tmp.0s48wMEdZl
+ rm /tmp/tmp.UTeBjzj5LF /tmp/tmp.0s48wMEdZl
+ return 0
2024-02-05T15:24:24.165Z	INFO	setup	Manager starting up	{"gitCommit": "5c1f635cea7dcb3dc6d1b248cb6465a7095a4f8e", "gitBranch": "PR-1588-5c1f635c", "buildTime": "2024-02-05T12:49:15Z", "goVersion": "go1.21.6", "os": "linux", "arch": "amd64"}
2024-02-05T15:24:24.165Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.25.16-gke.1360000"}
2024-02-05T15:24:24.166Z	INFO	setup	Registering Components.
2024-02-05T15:24:27.953Z	INFO	controller-runtime.metrics	Starting metrics server
2024-02-05T15:24:27.953Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2024-02-05T15:24:27.953Z	INFO	setup	Starting the Cmd.
2024-02-05T15:24:27.954Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2024-02-05T15:24:27.954Z	INFO	controller-runtime.webhook	Starting webhook server
2024-02-05T15:24:27.954Z	INFO	starting server	{"kind": "health probe", "addr": "[::]:8081"}
2024-02-05T15:24:27.955Z	INFO	attempting to acquire leader lease pxc-operator/08db0feb.percona.com...
2024-02-05T15:24:27.955Z	INFO	controller-runtime.certwatcher	Starting certificate watcher
2024-02-05T15:24:27.955Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate
2024-02-05T15:24:27.955Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2024-02-05T15:24:27.963Z	DEBUG	events	percona-xtradb-cluster-operator-66688f478-tqfbp_ac161d58-a682-4a21-8bbe-3cd48e3fe0bd became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db0feb.percona.com","uid":"537edb39-0f46-490d-9c50-923b07c2afc6","apiVersion":"coordination.k8s.io/v1","resourceVersion":"81974"}, "reason": "LeaderElection"}
2024-02-05T15:24:27.963Z	INFO	Starting Controller	{"controller": "pxcbackup-controller"}
2024-02-05T15:24:27.963Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2024-02-05T15:24:27.963Z	INFO	Starting Controller	{"controller": "pxcrestore-controller"}
2024-02-05T15:24:27.963Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2024-02-05T15:24:27.963Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2024-02-05T15:24:27.963Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2024-02-05T15:24:27.963Z	INFO	successfully acquired lease pxc-operator/08db0feb.percona.com
2024-02-05T15:24:28.144Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2024-02-05T15:24:28.147Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "worker count": 1}
2024-02-05T15:24:28.147Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "worker count": 1}
2024-02-05T15:24:52.466Z	INFO	Set CR version	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "f839c00c-2c08-4509-8253-abe94bdef799", "version": "1.14.0"}
2024-02-05T15:26:11.087Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "bde975e9-1090-4193-920b-7061afdec332", "user": "operator"}
2024-02-05T15:26:11.111Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "bde975e9-1090-4193-920b-7061afdec332", "user": "monitor"}
2024-02-05T15:26:11.160Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "bde975e9-1090-4193-920b-7061afdec332"}
2024-02-05T15:26:11.186Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "bde975e9-1090-4193-920b-7061afdec332"}
2024-02-05T15:26:11.209Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "bde975e9-1090-4193-920b-7061afdec332", "user": "xtrabackup"}
2024-02-05T15:26:11.241Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "bde975e9-1090-4193-920b-7061afdec332"}
2024-02-05T15:26:11.266Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "bde975e9-1090-4193-920b-7061afdec332", "user": "replication"}
2024-02-05T15:26:11.332Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "bde975e9-1090-4193-920b-7061afdec332", "err": "get primary pxc pod: not found"}
2024-02-05T15:26:16.153Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "1d9fb552-a36d-4562-b111-a645e2bccc2f", "err": "get primary pxc pod: not found"}
2024-02-05T15:26:21.407Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "2868366d-dbf6-4c23-b001-394ad35b6da4", "err": "get primary pxc pod: not found"}
2024-02-05T15:26:26.637Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "39cc9f24-8589-4beb-b03e-8b269d425e4c", "err": "get primary pxc pod: not found"}
2024-02-05T15:28:32.241Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "6859b330-8490-4967-a780-e85b1a12adc5", "user": "root"}
2024-02-05T15:28:32.482Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "6859b330-8490-4967-a780-e85b1a12adc5", "new version": "8.0.35-27.1"}
2024-02-05T15:28:34.386Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "6859b330-8490-4967-a780-e85b1a12adc5"}
2024-02-05T15:28:39.037Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "88e119d0-7902-4ad4-9f58-1494faaa8f3f"}
2024-02-05T15:28:44.420Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "45d48608-e3b2-4928-a297-79a84e38dcb6"}
2024-02-05T15:28:50.197Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "28c5f5db-c092-4113-8240-8fc9015958fd"}
2024-02-05T15:28:55.384Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "e9b05e7b-966d-4aee-be67-3c398c323d03"}
2024-02-05T15:29:00.620Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "d2c9f9db-9226-491e-957d-53774d97cf92"}
2024-02-05T15:29:05.984Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "8849c74c-a9f5-4534-b694-22da4539d35d"}
2024-02-05T15:29:11.397Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "fe6ea785-2ae8-4e69-a6e4-c9f3b90d95b3"}
2024-02-05T15:29:16.735Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "577daaa7-0989-4cd1-9e74-e675e68bc2b4"}
2024-02-05T15:29:22.428Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "d612513c-05e5-46c5-9920-98250713a6e9"}
2024-02-05T15:29:27.806Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "984af084-921d-457b-bbab-e8fa3a611161"}
2024-02-05T15:29:33.138Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "fdee0f57-f1ac-49e8-be1c-98499cf09ae2"}
2024-02-05T15:29:36.476Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c261b47d-cce1-4420-bf12-c70c8334c74c", "user": "root"}
2024-02-05T15:29:36.530Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c261b47d-cce1-4420-bf12-c70c8334c74c", "user": "root"}
2024-02-05T15:29:36.535Z	INFO	MySQL init secret created	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c261b47d-cce1-4420-bf12-c70c8334c74c", "secret": "some-name-mysql-init", "user": "root"}
2024-02-05T15:29:39.457Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c261b47d-cce1-4420-bf12-c70c8334c74c"}
2024-02-05T15:29:39.464Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c261b47d-cce1-4420-bf12-c70c8334c74c", "user": "root"}
2024-02-05T15:29:39.498Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c261b47d-cce1-4420-bf12-c70c8334c74c", "user": "root"}
2024-02-05T15:29:41.385Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c261b47d-cce1-4420-bf12-c70c8334c74c"}
2024-02-05T15:29:46.631Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "dc15b65c-1ac4-4284-bdf2-f3307de8f988"}
2024-02-05T15:29:52.102Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "bac5cbdc-7015-4471-b2aa-f743c7fb8f97"}
2024-02-05T15:29:54.722Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c565c6b2-31a2-497a-b06a-4bb5f81dca35", "error": "exec syncusers: pod some-name-proxysql-2 does not have a host assigned /  / ", "errorVerbose": "exec syncusers: pod some-name-proxysql-2 does not have a host assigned /  / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:30:13.829Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "0d35d4c0-dc83-4b68-82d3-da99d73257ff", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:387) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:387) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:30:18.629Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "6c8ab8fb-cfe6-49b0-8b63-26f849f2ca8d", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:387) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:387) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:30:21.851Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "91772ad8-5c24-477a-af14-f9673e5a0b20", "user": "proxyadmin"}
2024-02-05T15:30:21.851Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "91772ad8-5c24-477a-af14-f9673e5a0b20", "user": "proxyadmin"}
2024-02-05T15:30:21.907Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "91772ad8-5c24-477a-af14-f9673e5a0b20", "user": "proxyadmin"}
2024-02-05T15:30:21.916Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "91772ad8-5c24-477a-af14-f9673e5a0b20", "user": "proxyadmin"}
2024-02-05T15:30:21.917Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "91772ad8-5c24-477a-af14-f9673e5a0b20", "last-applied-secret": "1a55867d0f4fa0d0643653b2eb57b2f690315d8d843a10599c9839c2a779a95f"}
2024-02-05T15:30:22.117Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "91772ad8-5c24-477a-af14-f9673e5a0b20", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:31:07.814Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "7d1e4876-b5fb-4f76-907c-1c45af8a4a9f"}
2024-02-05T15:31:12.680Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "06854670-d0f1-4a7e-9669-05afcbbb3ee1"}
2024-02-05T15:31:17.997Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "32a73bbd-a089-4d5e-8076-436d72d8721f"}
2024-02-05T15:31:23.381Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "1e603f6a-1d7c-49c2-a286-250d3c85f142"}
2024-02-05T15:31:28.908Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "f2c107cf-5e14-4634-b1f9-0f71bc3624cc"}
2024-02-05T15:31:31.336Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "dedeb124-fa24-4655-94f0-606e0075219b"}
2024-02-05T15:31:32.393Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "8a14fe33-8364-4e24-95a8-96eba49264af", "user": "xtrabackup"}
2024-02-05T15:31:32.411Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "8a14fe33-8364-4e24-95a8-96eba49264af", "user": "xtrabackup"}
2024-02-05T15:31:32.417Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "8a14fe33-8364-4e24-95a8-96eba49264af", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-02-05T15:31:32.427Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "8a14fe33-8364-4e24-95a8-96eba49264af", "user": "xtrabackup"}
2024-02-05T15:31:32.457Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "8a14fe33-8364-4e24-95a8-96eba49264af", "user": "xtrabackup"}
2024-02-05T15:31:32.472Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "8a14fe33-8364-4e24-95a8-96eba49264af", "last-applied-secret": "d11bcc70ca53ea410cd455e13d82e7225917b0861d9b677ed34c02c5f6fbaf1b"}
2024-02-05T15:31:35.453Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "8a14fe33-8364-4e24-95a8-96eba49264af"}
2024-02-05T15:33:26.819Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "84052872-5721-4012-9ce1-f59ead6d5710", "err": "failed to connect to pod some-name-pxc-0: dial tcp 10.173.26.14:33062: connect: connection refused"}
2024-02-05T15:34:15.625Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "67a14850-d0bd-4ed4-965d-1f88ce9fa574"}
2024-02-05T15:34:20.682Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "3b011d57-2eb7-4fc0-bcc6-a1ea1321dc9d"}
2024-02-05T15:34:24.161Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "7218290b-2aff-4aa0-9f77-7ef39bbdee46", "user": "monitor"}
2024-02-05T15:34:24.181Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "7218290b-2aff-4aa0-9f77-7ef39bbdee46", "user": "monitor"}
2024-02-05T15:34:24.188Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "7218290b-2aff-4aa0-9f77-7ef39bbdee46", "secret": "some-name-mysql-init", "user": "monitor"}
2024-02-05T15:34:24.226Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "7218290b-2aff-4aa0-9f77-7ef39bbdee46", "user": "monitor"}
2024-02-05T15:34:24.233Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "7218290b-2aff-4aa0-9f77-7ef39bbdee46", "user": "monitor"}
2024-02-05T15:34:24.337Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "7218290b-2aff-4aa0-9f77-7ef39bbdee46", "last-applied-secret": "4e3282050745046d5044e0fcb5f2bf140d7320a1256a79c8ca2184f38f4f3098"}
2024-02-05T15:34:25.676Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "7218290b-2aff-4aa0-9f77-7ef39bbdee46", "error": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / ", "errorVerbose": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:35:08.177Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "323f95be-bb1c-4180-8e6e-c490751d479d", "user": "monitor"}
2024-02-05T15:35:10.203Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "323f95be-bb1c-4180-8e6e-c490751d479d"}
2024-02-05T15:35:13.148Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "9bd8065b-e7cb-4756-949a-ee08ba1e2584", "user": "monitor"}
2024-02-05T15:35:13.352Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "9bd8065b-e7cb-4756-949a-ee08ba1e2584", "user": "monitor"}
2024-02-05T15:35:13.380Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "9bd8065b-e7cb-4756-949a-ee08ba1e2584", "last-applied-secret": "4e3282050745046d5044e0fcb5f2bf140d7320a1256a79c8ca2184f38f4f3098"}
2024-02-05T15:35:15.105Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "9bd8065b-e7cb-4756-949a-ee08ba1e2584"}
2024-02-05T15:35:28.000Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "d6057d9a-2d61-4651-9892-5f91b8d3f365"}
2024-02-05T15:35:33.194Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "01c5bc33-ee85-4f11-b3de-96edcac89848"}
2024-02-05T15:35:38.592Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "895f294c-e1c6-47eb-8d71-3ec473049f2e"}
2024-02-05T15:35:42.097Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "509083fb-1bf2-42e1-9231-c134c2bcaa46", "user": "operator"}
2024-02-05T15:35:42.129Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "509083fb-1bf2-42e1-9231-c134c2bcaa46", "user": "operator"}
2024-02-05T15:35:42.136Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "509083fb-1bf2-42e1-9231-c134c2bcaa46", "secret": "some-name-mysql-init", "user": "operator"}
2024-02-05T15:35:42.143Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "509083fb-1bf2-42e1-9231-c134c2bcaa46", "user": "operator"}
2024-02-05T15:35:42.175Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "509083fb-1bf2-42e1-9231-c134c2bcaa46", "user": "operator"}
2024-02-05T15:35:42.211Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "509083fb-1bf2-42e1-9231-c134c2bcaa46", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-02-05T15:35:42.780Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "509083fb-1bf2-42e1-9231-c134c2bcaa46", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-29635.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-29635.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-29635.svc.' (using password: YES)\nERROR (line:1437) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-29635.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-29635.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-29635.svc.' (using password: YES)\nERROR (line:1437) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:36:07.245Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "a86b97c6-1b1c-474e-bea1-58b01bcb7ead"}
2024-02-05T15:36:11.181Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "624a35b5-74b9-4c97-81c7-201bad06940d"}
2024-02-05T15:36:16.590Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "e831d8a8-a117-4ad2-90a4-1d9bbb88f1b2"}
2024-02-05T15:36:22.177Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "e401c640-3dea-41bb-9929-41586b10d746"}
2024-02-05T15:36:27.434Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "d2c3fc95-bcee-46b7-bfa9-4cb61e15a7ec"}
2024-02-05T15:36:32.807Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "1da1fa69-38cc-485e-af59-ffa6abf831c3"}
2024-02-05T15:36:38.117Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "8ff9fea3-9dea-4d7a-b7ba-76f9422aa62c"}
2024-02-05T15:36:38.674Z	INFO	Created user secrets	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "secrets": "my-cluster-secrets-2"}
2024-02-05T15:36:38.690Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "root"}
2024-02-05T15:36:38.715Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "root"}
2024-02-05T15:36:38.723Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "secret": "some-name-mysql-init", "user": "root"}
2024-02-05T15:36:41.213Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d"}
2024-02-05T15:36:41.221Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "root"}
2024-02-05T15:36:41.252Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "root"}
2024-02-05T15:36:41.265Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "operator"}
2024-02-05T15:36:41.295Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "operator"}
2024-02-05T15:36:41.301Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "secret": "some-name-mysql-init", "user": "operator"}
2024-02-05T15:36:41.310Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "operator"}
2024-02-05T15:36:41.327Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "operator"}
2024-02-05T15:36:41.337Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "monitor"}
2024-02-05T15:36:41.363Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "monitor"}
2024-02-05T15:36:41.368Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "secret": "some-name-mysql-init", "user": "monitor"}
2024-02-05T15:36:41.397Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "monitor"}
2024-02-05T15:36:41.404Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "monitor"}
2024-02-05T15:36:41.474Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "xtrabackup"}
2024-02-05T15:36:41.493Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "xtrabackup"}
2024-02-05T15:36:41.500Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-02-05T15:36:41.506Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "xtrabackup"}
2024-02-05T15:36:41.537Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "xtrabackup"}
2024-02-05T15:36:41.551Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "replication"}
2024-02-05T15:36:41.575Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "replication"}
2024-02-05T15:36:41.582Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "secret": "some-name-mysql-init", "user": "replication"}
2024-02-05T15:36:41.590Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "replication"}
2024-02-05T15:36:41.611Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "replication"}
2024-02-05T15:36:41.611Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "proxyadmin"}
2024-02-05T15:36:41.645Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "proxyadmin"}
2024-02-05T15:36:41.656Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "user": "proxyadmin"}
2024-02-05T15:36:41.656Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "last-applied-secret": "e2224dbbc7399f5d510fe6752c4417e5cf71e935cb0e9f60974e6dcaa900b380"}
2024-02-05T15:36:41.656Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "last-applied-secret": "e2224dbbc7399f5d510fe6752c4417e5cf71e935cb0e9f60974e6dcaa900b380"}
2024-02-05T15:36:41.878Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9f832aa-ba4e-4bd6-a0ef-711d7fb2670d", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:36:42.153Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "be19a11c-19e3-472b-b7ab-64961e320852", "user": "monitor"}
2024-02-05T15:36:42.456Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "be19a11c-19e3-472b-b7ab-64961e320852", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:37:34.626Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "517b5c6a-4d28-4c0c-a900-5b07359351e5", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-29635 on 10.173.32.10:53: no such host"}
2024-02-05T15:37:39.678Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "7d1b786c-6436-4cbf-9f21-7d109753ff61", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-29635 on 10.173.32.10:53: no such host"}
2024-02-05T15:37:44.927Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "de1f0bbd-4ce4-4039-b0e7-fcea30bb5ed9", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-29635 on 10.173.32.10:53: no such host"}
2024-02-05T15:38:37.194Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "b04d6731-0c6a-49f7-9e2c-2956d48d4c71", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-29635 on 10.173.32.10:53: no such host"}
2024-02-05T15:38:42.332Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "9f203cc2-c698-43bb-bb46-1f7a481e50ff", "primary name": "some-name-pxc-0.some-name-pxc.users-29635.svc.cluster.local"}
2024-02-05T15:38:47.458Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c9763913-d1c9-4d63-9284-a2616d202005", "primary name": "some-name-pxc-0.some-name-pxc.users-29635.svc.cluster.local"}
2024-02-05T15:38:52.582Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "6648a164-a981-4ed4-afa7-14ca6b7a7978", "primary name": "some-name-pxc-0.some-name-pxc.users-29635.svc.cluster.local"}
2024-02-05T15:38:57.718Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "3ec302e4-2ff3-491a-affe-3d4f67428b8f", "primary name": "some-name-pxc-0.some-name-pxc.users-29635.svc.cluster.local"}
2024-02-05T15:39:18.917Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "6419c4c1-d0c9-471d-b931-9eda8de07312", "user": "monitor"}
2024-02-05T15:39:19.132Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "6419c4c1-d0c9-471d-b931-9eda8de07312", "user": "monitor"}
2024-02-05T15:39:19.158Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "6419c4c1-d0c9-471d-b931-9eda8de07312", "last-applied-secret": "e2224dbbc7399f5d510fe6752c4417e5cf71e935cb0e9f60974e6dcaa900b380"}
2024-02-05T15:39:20.796Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "6419c4c1-d0c9-471d-b931-9eda8de07312"}
2024-02-05T15:39:25.706Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "f3a77c2b-9e6d-4c13-b010-67bb4304fad5"}
2024-02-05T15:39:29.257Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ff002d88-9db5-46a8-b512-f9dde6fdd79b", "user": "operator"}
2024-02-05T15:39:29.290Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ff002d88-9db5-46a8-b512-f9dde6fdd79b", "user": "operator"}
2024-02-05T15:39:29.297Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ff002d88-9db5-46a8-b512-f9dde6fdd79b", "secret": "some-name-mysql-init", "user": "operator"}
2024-02-05T15:39:29.306Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ff002d88-9db5-46a8-b512-f9dde6fdd79b", "user": "operator"}
2024-02-05T15:39:29.328Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ff002d88-9db5-46a8-b512-f9dde6fdd79b", "user": "operator"}
2024-02-05T15:39:29.372Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ff002d88-9db5-46a8-b512-f9dde6fdd79b", "last-applied-secret": "1fe4bcf8c1e9aedaca63e64a41d47e5ffd14e9a5d51750b02cb23e4c8d1f0bf7"}
2024-02-05T15:39:29.918Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ff002d88-9db5-46a8-b512-f9dde6fdd79b", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-29635.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-29635.svc.' (using password: YES)\nERROR (line:1437) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-29635.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-29635.svc.' (using password: YES)\nERROR (line:1437) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:40:09.898Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "4f12e51c-497c-42b4-a7ad-8d2f96303126"}
2024-02-05T15:40:13.934Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "31386c71-23b1-4379-b1de-5c7c147be545"}
2024-02-05T15:40:19.205Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "53cd90f1-e2b3-4434-9151-61b61e4ad914"}
2024-02-05T15:40:24.522Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "025a9ac3-b9cc-4151-8c5f-160d59bc0062"}
2024-02-05T15:40:29.811Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "b0cf3fe8-4bef-4d7d-991f-0ea7167afae3"}
2024-02-05T15:40:35.119Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "0891d942-43ce-4f68-838a-d808b8aad9aa"}
2024-02-05T15:40:40.915Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "946f7a57-a0f5-4e1b-9d3b-48815193df06"}
2024-02-05T15:40:46.285Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "f3ba4a2f-3ecb-4d14-b53c-5979d820e2b1"}
2024-02-05T15:40:51.229Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "2b87a781-edfb-4646-bfa3-6ce5b2571d18"}
2024-02-05T15:40:56.494Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "803c4059-c92c-4ec7-8ada-c999a946cbb9"}
2024-02-05T15:41:01.828Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "15f24fda-8b8f-43ae-84c5-50c090e3014e"}
2024-02-05T15:41:07.781Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "bafc0161-581a-458d-bd41-629e3c650b87"}
2024-02-05T15:41:12.994Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "88da89a3-105b-4107-9e35-1f7e49d87e02"}
2024-02-05T15:41:18.392Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "c7425a22-7f31-4380-9cc2-b09c066aaa99"}
2024-02-05T15:41:23.716Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "57b1c884-7f4d-40e2-9b4a-a56bdd24495e"}
2024-02-05T15:41:27.026Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "root"}
2024-02-05T15:41:27.070Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "root"}
2024-02-05T15:41:27.075Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "secret": "some-name-mysql-init", "user": "root"}
2024-02-05T15:41:29.713Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d"}
2024-02-05T15:41:29.723Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "root"}
2024-02-05T15:41:29.751Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "root"}
2024-02-05T15:41:29.776Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "monitor"}
2024-02-05T15:41:29.793Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "monitor"}
2024-02-05T15:41:29.804Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "secret": "some-name-mysql-init", "user": "monitor"}
2024-02-05T15:41:29.833Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "monitor"}
2024-02-05T15:41:29.843Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "monitor"}
2024-02-05T15:41:29.921Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "xtrabackup"}
2024-02-05T15:41:29.941Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "xtrabackup"}
2024-02-05T15:41:29.952Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-02-05T15:41:29.961Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "xtrabackup"}
2024-02-05T15:41:29.984Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "xtrabackup"}
2024-02-05T15:41:29.995Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "proxyadmin"}
2024-02-05T15:41:30.025Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "proxyadmin"}
2024-02-05T15:41:30.036Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "user": "proxyadmin"}
2024-02-05T15:41:30.036Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "last-applied-secret": "88e787b8abed829afdbe5205d0e5eebf506898b166c9f67d9d02eb3d30ca21ed"}
2024-02-05T15:41:30.036Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "last-applied-secret": "88e787b8abed829afdbe5205d0e5eebf506898b166c9f67d9d02eb3d30ca21ed"}
2024-02-05T15:41:30.197Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ef7e9f89-3553-4932-81a3-b3a8df46303d", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:41:50.647Z	INFO	KubeAPIWarningLogger	.metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 02ff43e8-9189-45db-85c0-7ea4ede9fa9d
2024-02-05T15:43:29.328Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "d5764969-eb3d-41a3-b10f-706898e9ddb5", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp 10.173.25.22:33062: connect: connection refused"}
2024-02-05T15:44:36.965Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "94a8e22c-068b-4c79-afe8-5c572aeca7b0", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-29635 on 10.173.32.10:53: no such host"}
2024-02-05T15:45:03.498Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "root"}
2024-02-05T15:45:03.527Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "root"}
2024-02-05T15:45:03.537Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "secret": "some-name-mysql-init", "user": "root"}
2024-02-05T15:45:03.549Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "root"}
2024-02-05T15:45:03.581Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "root"}
2024-02-05T15:45:03.595Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "operator"}
2024-02-05T15:45:03.623Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "operator"}
2024-02-05T15:45:03.632Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "secret": "some-name-mysql-init", "user": "operator"}
2024-02-05T15:45:03.639Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "operator"}
2024-02-05T15:45:03.668Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "operator"}
2024-02-05T15:45:03.678Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "monitor"}
2024-02-05T15:45:03.697Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "monitor"}
2024-02-05T15:45:03.702Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "secret": "some-name-mysql-init", "user": "monitor"}
2024-02-05T15:45:03.709Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "monitor"}
2024-02-05T15:45:03.787Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "xtrabackup"}
2024-02-05T15:45:03.805Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "xtrabackup"}
2024-02-05T15:45:03.813Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-02-05T15:45:03.822Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "xtrabackup"}
2024-02-05T15:45:03.849Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "xtrabackup"}
2024-02-05T15:45:03.860Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "replication"}
2024-02-05T15:45:03.879Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "replication"}
2024-02-05T15:45:03.886Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "secret": "some-name-mysql-init", "user": "replication"}
2024-02-05T15:45:03.893Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "replication"}
2024-02-05T15:45:03.913Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "user": "replication"}
2024-02-05T15:45:03.913Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-02-05T15:45:03.913Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "74a821f4-a99a-499d-9e05-3ef50163a4dc", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-02-05T15:46:16.333Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "ad3c7b79-74ef-4c19-af64-a32e5143b934", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-29635 on 10.173.32.10:53: no such host"}
2024-02-05T15:47:34.710Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "d89b5c90-cae4-45d2-bdd9-68d744f75be2", "user": "monitor"}
2024-02-05T15:47:35.077Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "d89b5c90-cae4-45d2-bdd9-68d744f75be2", "user": "monitor"}
2024-02-05T15:47:35.100Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "d89b5c90-cae4-45d2-bdd9-68d744f75be2", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-02-05T15:47:50.256Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "23e26b1e-1eb4-422d-b389-84046d8ed931", "user": "monitor"}
2024-02-05T15:47:50.283Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "23e26b1e-1eb4-422d-b389-84046d8ed931", "user": "monitor"}
2024-02-05T15:47:50.292Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "23e26b1e-1eb4-422d-b389-84046d8ed931", "secret": "some-name-mysql-init", "user": "monitor"}
2024-02-05T15:47:50.301Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "23e26b1e-1eb4-422d-b389-84046d8ed931", "user": "monitor"}
2024-02-05T15:47:50.410Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "23e26b1e-1eb4-422d-b389-84046d8ed931", "last-applied-secret": "81935ecf30ae523a6ee69582354f9488d47dbeb0f8c7c572af96870f2024284d"}
2024-02-05T15:48:59.450Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "5ba3e7e4-67cc-4a0c-b30f-9dda19200c1f", "user": "monitor"}
2024-02-05T15:48:59.717Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "5ba3e7e4-67cc-4a0c-b30f-9dda19200c1f", "user": "monitor"}
2024-02-05T15:48:59.748Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-29635", "name": "some-name", "reconcileID": "5ba3e7e4-67cc-4a0c-b30f-9dda19200c1f", "last-applied-secret": "81935ecf30ae523a6ee69582354f9488d47dbeb0f8c7c572af96870f2024284d"}
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:227
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:266
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:329
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1235
[mysql] 2024/02/05 15:31:29 packets.go:37: unexpected EOF
[mysql] 2024/02/05 15:33:26 packets.go:37: read tcp 10.173.25.13:54786->10.173.26.14:33062: read: connection reset by peer
[mysql] 2024/02/05 15:34:24 packets.go:37: unexpected EOF
[mysql] 2024/02/05 15:34:55 packets.go:37: unexpected EOF
[mysql] 2024/02/05 15:44:21 packets.go:37: unexpected EOF
[mysql] 2024/02/05 15:44:36 packets.go:37: read tcp 10.173.25.13:51954->10.173.45.78:3306: i/o timeout
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get pxc --all-namespaces -o wide
+ kubectl patch pxc -n users-29635 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.dkYmI1xV9e
++ mktemp
+ local LAST_ERR=/tmp/tmp.nrpZIA89Kt
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.dkYmI1xV9e
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.nrpZIA89Kt
+ rm /tmp/tmp.dkYmI1xV9e /tmp/tmp.nrpZIA89Kt
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.F6VeMijUms
++ mktemp
+ local LAST_ERR=/tmp/tmp.ynSACVN1ig
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.F6VeMijUms
No resources found
+ cat /tmp/tmp.ynSACVN1ig
+ rm /tmp/tmp.F6VeMijUms /tmp/tmp.ynSACVN1ig
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.cAjnpap5SR
++ mktemp
+ local LAST_ERR=/tmp/tmp.5Lv2pceO1K
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.cAjnpap5SR
No resources found
+ cat /tmp/tmp.5Lv2pceO1K
+ rm /tmp/tmp.cAjnpap5SR /tmp/tmp.5Lv2pceO1K
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.I9g7wpEXxd
++ mktemp
+ local LAST_ERR=/tmp/tmp.t2bGTZBbgt
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.I9g7wpEXxd
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.t2bGTZBbgt
+ rm /tmp/tmp.I9g7wpEXxd /tmp/tmp.t2bGTZBbgt
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.12.1/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace users-29635
+ rm -rf /tmp/tmp.T7soxWEpSM
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ local LAST_OUT=/tmp/tmp.Hnp2kxTbrw
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.ytvk4M9u6f
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.Yf0xpUeZo8
+ local LAST_ERR=/tmp/tmp.ds6xOx3sb0
+ local exit_status=0
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace users-29635