Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/logs/users-5-7.log
WARNING: version difference between client (1.30) and server (1.27) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.27) exceeds the supported minor version skew of +/-1
+ create_infra users-16820
+ local ns=users-16820
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n users-24891 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.kJ95EZBNPW
++ mktemp
+ local LAST_ERR=/tmp/tmp.jyigpt8q8A
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kJ95EZBNPW
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.jyigpt8q8A
+ rm /tmp/tmp.kJ95EZBNPW /tmp/tmp.jyigpt8q8A
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.ausAdMUAIE
++ mktemp
+ local LAST_ERR=/tmp/tmp.b5zQVBqIul
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ausAdMUAIE
No resources found
+ cat /tmp/tmp.b5zQVBqIul
+ rm /tmp/tmp.ausAdMUAIE /tmp/tmp.b5zQVBqIul
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.5l62FH8Hnq
++ mktemp
+ local LAST_ERR=/tmp/tmp.WrRE60MyYX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5l62FH8Hnq
No resources found
+ cat /tmp/tmp.WrRE60MyYX
+ rm /tmp/tmp.5l62FH8Hnq /tmp/tmp.WrRE60MyYX
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.xEjflBGF0E
++ mktemp
++ mktemp
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ local LAST_OUT=/tmp/tmp.8ZhGGCBK9H
+ local LAST_ERR=/tmp/tmp.GT24bTPCuH
+ local exit_status=0
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.KtBBetESdv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xEjflBGF0E
+ cat /tmp/tmp.GT24bTPCuH
+ rm /tmp/tmp.xEjflBGF0E /tmp/tmp.GT24bTPCuH
+ return 0
namespace "gmp-public" deleted
namespace "gmp-system" deleted
namespace "users-24891" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8ZhGGCBK9H
namespace "pxc-operator" deleted
+ cat /tmp/tmp.KtBBetESdv
+ rm /tmp/tmp.8ZhGGCBK9H /tmp/tmp.KtBBetESdv
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'namespace/pxc-operator - '
namespace/pxc-operator - + set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.D1F5aalQUl
++ mktemp
+ local LAST_ERR=/tmp/tmp.iDsWGN88d4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.D1F5aalQUl
namespace/pxc-operator created
+ cat /tmp/tmp.iDsWGN88d4
+ rm /tmp/tmp.D1F5aalQUl /tmp/tmp.iDsWGN88d4
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lsrMeVOYxr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DUlKCGyWPw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.lsrMeVOYxr
++ cat /tmp/tmp.DUlKCGyWPw
++ rm /tmp/tmp.lsrMeVOYxr /tmp/tmp.DUlKCGyWPw
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1753-50eda1a3-33-cluster5 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.SBTMTToXCh
++ mktemp
+ local LAST_ERR=/tmp/tmp.NXwUrChO7V
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1753-50eda1a3-33-cluster5 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.SBTMTToXCh
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1753-50eda1a3-33-cluster5" modified.
+ cat /tmp/tmp.NXwUrChO7V
+ rm /tmp/tmp.SBTMTToXCh /tmp/tmp.NXwUrChO7V
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.AIE0qOXxFq
++ mktemp
+ local LAST_ERR=/tmp/tmp.hs3il1GRr6
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.AIE0qOXxFq
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.hs3il1GRr6
+ rm /tmp/tmp.AIE0qOXxFq /tmp/tmp.hs3il1GRr6
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.EMJmOTamR9
++ mktemp
+ local LAST_ERR=/tmp/tmp.mLEEIhlS7Q
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EMJmOTamR9
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.mLEEIhlS7Q
+ rm /tmp/tmp.EMJmOTamR9 /tmp/tmp.mLEEIhlS7Q
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3^'
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.6f5srAx4te
++ mktemp
+ local LAST_ERR=/tmp/tmp.JjngFA0jg4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6f5srAx4te
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.JjngFA0jg4
+ rm /tmp/tmp.6f5srAx4te /tmp/tmp.JjngFA0jg4
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.nmzgkZB2tj
++ mktemp
+ local LAST_ERR=/tmp/tmp.s8dmA5SUBT
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nmzgkZB2tj
pod/percona-xtradb-cluster-operator-6f69b446fd-lkmvc condition met
+ cat /tmp/tmp.s8dmA5SUBT
+ rm /tmp/tmp.nmzgkZB2tj /tmp/tmp.s8dmA5SUBT
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.s4AqcT8dXa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hrdCgGIZh3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.s4AqcT8dXa
++ cat /tmp/tmp.hrdCgGIZh3
++ rm /tmp/tmp.s4AqcT8dXa /tmp/tmp.hrdCgGIZh3
++ return 0
+ wait_pod percona-xtradb-cluster-operator-6f69b446fd-lkmvc 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-6f69b446fd-lkmvc
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-6f69b446fd-lkmvc
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-6f69b446fd-lkmvc condition met
percona-xtradb-cluster-operator-6f69b446fd-lkmvc.Ok
+ sleep 3
+ create_namespace users-16820
+ local namespace=users-16820
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces users-16820'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces users-16820
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace users-16820
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.z9Zo2mT0p4
+ local LAST_OUT=/tmp/tmp.a7QGKTyKbl
++ mktemp
+ local LAST_ERR=/tmp/tmp.VpSokmqpE7
+ local exit_status=0
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.jzfZg6tu9c
+ local exit_status=0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-16820
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-16820
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.z9Zo2mT0p4
+ cat /tmp/tmp.VpSokmqpE7
+ rm /tmp/tmp.z9Zo2mT0p4 /tmp/tmp.VpSokmqpE7
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-16820
namespace "gmp-public" deleted
namespace "gmp-system" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.a7QGKTyKbl
+ cat /tmp/tmp.jzfZg6tu9c
Error from server (NotFound): namespaces "users-16820" not found
+ rm /tmp/tmp.a7QGKTyKbl /tmp/tmp.jzfZg6tu9c
+ return 1
+ :
+ wait_for_delete namespace/users-16820
+ local res=namespace/users-16820
+ echo -n 'namespace/users-16820 - '
namespace/users-16820 - + set +o xtrace
Error from server (NotFound): namespaces "users-16820" not found
+ desc 'create namespace users-16820'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace users-16820
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace users-16820
++ mktemp
+ local LAST_OUT=/tmp/tmp.g7ySFUXb0A
++ mktemp
+ local LAST_ERR=/tmp/tmp.AVopJoQTp4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace users-16820
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.g7ySFUXb0A
namespace/users-16820 created
+ cat /tmp/tmp.AVopJoQTp4
+ rm /tmp/tmp.g7ySFUXb0A /tmp/tmp.AVopJoQTp4
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jopsPliCUo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QgWRVWH6ji
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.jopsPliCUo
++ cat /tmp/tmp.QgWRVWH6ji
++ rm /tmp/tmp.jopsPliCUo /tmp/tmp.QgWRVWH6ji
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1753-50eda1a3-33-cluster5 --namespace=users-16820
++ mktemp
+ local LAST_OUT=/tmp/tmp.7DRHwoQBpp
++ mktemp
+ local LAST_ERR=/tmp/tmp.lsDioXGRDd
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1753-50eda1a3-33-cluster5 --namespace=users-16820
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7DRHwoQBpp
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1753-50eda1a3-33-cluster5" modified.
+ cat /tmp/tmp.lsDioXGRDd
+ rm /tmp/tmp.7DRHwoQBpp /tmp/tmp.lsDioXGRDd
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ix4gWTQ0JI
++ mktemp
+ local LAST_ERR=/tmp/tmp.cL6gQpNun7
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ix4gWTQ0JI
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.cL6gQpNun7
+ rm /tmp/tmp.ix4gWTQ0JI /tmp/tmp.cL6gQpNun7
+ return 0
+ desc 'create PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC cluster
-----------------------------------------------------------------------------------

+ newpass=test-password
++ echo -n test-password
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZA==
+ cluster=some-name
+ spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/some-name.yml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/some-name.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.qbcKmRf2hi
++ mktemp
+ local LAST_ERR=/tmp/tmp.QYAenwMhEU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.qbcKmRf2hi
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.QYAenwMhEU
+ rm /tmp/tmp.qbcKmRf2hi /tmp/tmp.QYAenwMhEU
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-16820~
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ local LAST_OUT=/tmp/tmp.MzaLmM64FL
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3#'
++ mktemp
+ local LAST_ERR=/tmp/tmp.ELpW1gtdhz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MzaLmM64FL
deployment.apps/pxc-client created
+ cat /tmp/tmp.ELpW1gtdhz
+ rm /tmp/tmp.MzaLmM64FL /tmp/tmp.ELpW1gtdhz
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ some-name == \d\e\m\a\n\d\-\b\a\c\k\u\p ]]
+ [[ some-name == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/some-name.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/some-name.yml
++ mktemp
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/conf/some-name.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ local LAST_OUT=/tmp/tmp.H85IQEdl3I
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-16820~
+ local LAST_ERR=/tmp/tmp.SX36CVVhLp
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.H85IQEdl3I
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.SX36CVVhLp
+ rm /tmp/tmp.H85IQEdl3I /tmp/tmp.SX36CVVhLp
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy some-name
++ local target_cluster=some-name
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.GKnD036gmR
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cdvyPdsZxN
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.GKnD036gmR
+++ cat /tmp/tmp.cdvyPdsZxN
+++ rm /tmp/tmp.GKnD036gmR /tmp/tmp.cdvyPdsZxN
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yEaiok5F08
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ODhMy6YVFy
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.yEaiok5F08
+++ cat /tmp/tmp.ODhMy6YVFy
+++ rm /tmp/tmp.yEaiok5F08 /tmp/tmp.ODhMy6YVFy
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo some-name-proxysql
++ return
+ local proxy=some-name-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-16820
++ mktemp
+ local LAST_OUT=/tmp/tmp.2VHIrXsJD0
++ mktemp
+ local LAST_ERR=/tmp/tmp.2PiEI7Y0ou
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-16820
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-16820
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-16820
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.2VHIrXsJD0
+ cat /tmp/tmp.2PiEI7Y0ou
error: no matching resources found
+ rm /tmp/tmp.2VHIrXsJD0 /tmp/tmp.2PiEI7Y0ou
+ return 1
+ true
+ wait_for_running some-name-proxysql 1
+ local name=some-name-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-proxysql-0 480
+ local pod=some-name-proxysql-0
+ local max_retry=480
+ local ns=
++ echo some-name-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
pod/some-name-proxysql-0 condition met
some-name-proxysql-0.Ok
+ wait_for_running some-name-pxc 3
+ local name=some-name-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
some-name-pxc-0.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-1 480
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
some-name-pxc-1.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-2 480
+ local pod=some-name-pxc-2
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-2 condition met
some-name-pxc-2.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
++ is_keyring_plugin_in_use some-name
++ local cluster=some-name
++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yDXyYeh8Jx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GBt2xUxzI9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.yDXyYeh8Jx
++ cat /tmp/tmp.GBt2xUxzI9
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.yDXyYeh8Jx /tmp/tmp.GBt2xUxzI9
++ return 0
+ [[ -n '' ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-proxysql -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h some-name-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YA2psixA2q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r72L4MeEhk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YA2psixA2q
++ cat /tmp/tmp.r72L4MeEhk
++ rm /tmp/tmp.YA2psixA2q /tmp/tmp.r72L4MeEhk
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ echo pxc-client-7b7f8bcff9-gg4pr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-proxysql -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h some-name-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aeCCRbjVs6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.V5QweAoQIq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.aeCCRbjVs6
++ cat /tmp/tmp.V5QweAoQIq
++ rm /tmp/tmp.aeCCRbjVs6 /tmp/tmp.V5QweAoQIq
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b7f8bcff9-gg4pr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.crpvy20ejg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5qFtZaboRD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.crpvy20ejg
++ cat /tmp/tmp.5qFtZaboRD
++ rm /tmp/tmp.crpvy20ejg /tmp/tmp.5qFtZaboRD
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b7f8bcff9-gg4pr
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-1.sql /tmp/tmp.x1AMCGp4R5/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IjZizfcDwd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lSJUoWUuQI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IjZizfcDwd
++ cat /tmp/tmp.lSJUoWUuQI
++ rm /tmp/tmp.IjZizfcDwd /tmp/tmp.lSJUoWUuQI
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b7f8bcff9-gg4pr
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-1.sql /tmp/tmp.x1AMCGp4R5/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TpaWSiH5jY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GNn92fGo1f
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TpaWSiH5jY
++ cat /tmp/tmp.GNn92fGo1f
++ rm /tmp/tmp.TpaWSiH5jY /tmp/tmp.GNn92fGo1f
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b7f8bcff9-gg4pr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-1.sql /tmp/tmp.x1AMCGp4R5/select-1.sql
++ is_keyring_plugin_in_use some-name
++ local cluster=some-name
++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Xl2v7x6bP1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aYnIgCMgK7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Xl2v7x6bP1
++ cat /tmp/tmp.aYnIgCMgK7
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.Xl2v7x6bP1 /tmp/tmp.aYnIgCMgK7
++ return 0
+ '[' '' ']'
+ desc 'test root'
+ set +o xtrace


-----------------------------------------------------------------------------------
test root
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets root dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=root
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.pCXv1yVSW3
++ mktemp
+ local LAST_ERR=/tmp/tmp.T83QkXHYtU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.pCXv1yVSW3
secret/my-cluster-secrets patched
+ cat /tmp/tmp.T83QkXHYtU
+ rm /tmp/tmp.pCXv1yVSW3 /tmp/tmp.T83QkXHYtU
+ return 0
+ sleep 15
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xGcllv44hP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mxrNT3Zkga
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xGcllv44hP
++ cat /tmp/tmp.mxrNT3Zkga
++ rm /tmp/tmp.xGcllv44hP /tmp/tmp.mxrNT3Zkga
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b7f8bcff9-gg4pr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-4.sql /tmp/tmp.x1AMCGp4R5/select-4.sql
+ desc 'test proxyadmin'
+ set +o xtrace


-----------------------------------------------------------------------------------
test proxyadmin
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.12VMoqZzLV
++ mktemp
+ local LAST_ERR=/tmp/tmp.1F5n48LrmF
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.12VMoqZzLV
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.1F5n48LrmF
+ rm /tmp/tmp.12VMoqZzLV /tmp/tmp.1F5n48LrmF
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GDx5qgMWTZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hREanPiHow
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GDx5qgMWTZ
++ cat /tmp/tmp.hREanPiHow
++ rm /tmp/tmp.GDx5qgMWTZ /tmp/tmp.hREanPiHow
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kQgK1HkneS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TrQojkUb10
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kQgK1HkneS
++ cat /tmp/tmp.TrQojkUb10
++ rm /tmp/tmp.kQgK1HkneS /tmp/tmp.TrQojkUb10
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hGfugfwF9s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EB7GusJrsA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hGfugfwF9s
++ cat /tmp/tmp.EB7GusJrsA
++ rm /tmp/tmp.hGfugfwF9s /tmp/tmp.EB7GusJrsA
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.sDEvQMtTXN
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.a94cNPixuP
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.sDEvQMtTXN
+++++ cat /tmp/tmp.a94cNPixuP
+++++ rm /tmp/tmp.sDEvQMtTXN /tmp/tmp.a94cNPixuP
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.arwU0YXJtM
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.axSYxpKAZf
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.arwU0YXJtM
+++++ cat /tmp/tmp.axSYxpKAZf
+++++ rm /tmp/tmp.arwU0YXJtM /tmp/tmp.axSYxpKAZf
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ddCVuJruao
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NN14pbkSI7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ddCVuJruao
++ cat /tmp/tmp.NN14pbkSI7
++ rm /tmp/tmp.ddCVuJruao /tmp/tmp.NN14pbkSI7
++ return 0
+ [[ 3 == \3 ]]
+ patch_secret my-cluster-secrets proxyadmin dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=proxyadmin
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.zDrVOfBVZh
++ mktemp
+ local LAST_ERR=/tmp/tmp.EKGBYvJni0
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.zDrVOfBVZh
secret/my-cluster-secrets patched
+ cat /tmp/tmp.EKGBYvJni0
+ rm /tmp/tmp.zDrVOfBVZh /tmp/tmp.EKGBYvJni0
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HzvvxK64GR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gs3loN2Ttj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.HzvvxK64GR
++ cat /tmp/tmp.gs3loN2Ttj
++ rm /tmp/tmp.HzvvxK64GR /tmp/tmp.gs3loN2Ttj
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MQ4fhDgmdg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Nkj7bcKkS1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.MQ4fhDgmdg
++ cat /tmp/tmp.Nkj7bcKkS1
++ rm /tmp/tmp.MQ4fhDgmdg /tmp/tmp.Nkj7bcKkS1
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.45cgYOB3EF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Y6MxOu5kX2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.45cgYOB3EF
++ cat /tmp/tmp.Y6MxOu5kX2
++ rm /tmp/tmp.45cgYOB3EF /tmp/tmp.Y6MxOu5kX2
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.J9b3e3LDdc
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.EX3sIFt9O1
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.J9b3e3LDdc
+++++ cat /tmp/tmp.EX3sIFt9O1
+++++ rm /tmp/tmp.J9b3e3LDdc /tmp/tmp.EX3sIFt9O1
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.nvz4skKflw
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.pYL58GeB6t
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.nvz4skKflw
+++++ cat /tmp/tmp.pYL58GeB6t
+++++ rm /tmp/tmp.nvz4skKflw /tmp/tmp.pYL58GeB6t
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aFJnEHu1Dv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MRe3IYrG1a
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.aFJnEHu1Dv
++ cat /tmp/tmp.MRe3IYrG1a
++ rm /tmp/tmp.aFJnEHu1Dv /tmp/tmp.MRe3IYrG1a
++ return 0
+ [[ 3 == \3 ]]
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-0
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-0
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-2.sql /tmp/tmp.x1AMCGp4R5/select-2.sql
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-1
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-1
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-2.sql /tmp/tmp.x1AMCGp4R5/select-2.sql
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-2
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-2
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-2.sql /tmp/tmp.x1AMCGp4R5/select-2.sql
+ desc 'test xtrabackup'
+ set +o xtrace


-----------------------------------------------------------------------------------
test xtrabackup
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.mzH9Vohcia
++ mktemp
+ local LAST_ERR=/tmp/tmp.WazkdJ6QTi
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.mzH9Vohcia
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.WazkdJ6QTi
+ rm /tmp/tmp.mzH9Vohcia /tmp/tmp.WazkdJ6QTi
+ return 0
+ patch_secret my-cluster-secrets xtrabackup dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=xtrabackup
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.2hJmrjhHzu
++ mktemp
+ local LAST_ERR=/tmp/tmp.yGgjoblKLG
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2hJmrjhHzu
secret/my-cluster-secrets patched
+ cat /tmp/tmp.yGgjoblKLG
+ rm /tmp/tmp.2hJmrjhHzu /tmp/tmp.yGgjoblKLG
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9xDtq6nZxT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TaI6AGhpWS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9xDtq6nZxT
++ cat /tmp/tmp.TaI6AGhpWS
++ rm /tmp/tmp.9xDtq6nZxT /tmp/tmp.TaI6AGhpWS
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rhl8TMZHEL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ntvsyE9Zul
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rhl8TMZHEL
++ cat /tmp/tmp.ntvsyE9Zul
++ rm /tmp/tmp.rhl8TMZHEL /tmp/tmp.ntvsyE9Zul
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DtpzwnMbzI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8o38aJROvF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DtpzwnMbzI
++ cat /tmp/tmp.8o38aJROvF
++ rm /tmp/tmp.DtpzwnMbzI /tmp/tmp.8o38aJROvF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OkQjW22Hbk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YbhKrUbWc3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.OkQjW22Hbk
++ cat /tmp/tmp.YbhKrUbWc3
++ rm /tmp/tmp.OkQjW22Hbk /tmp/tmp.YbhKrUbWc3
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SOR1KTJd1e
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2dvCSb2AZ1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SOR1KTJd1e
++ cat /tmp/tmp.2dvCSb2AZ1
++ rm /tmp/tmp.SOR1KTJd1e /tmp/tmp.2dvCSb2AZ1
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rsuy9hGPR8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gDfu6VQPmh
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rsuy9hGPR8
++ cat /tmp/tmp.gDfu6VQPmh
++ rm /tmp/tmp.rsuy9hGPR8 /tmp/tmp.gDfu6VQPmh
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2c8NjXIDgk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EWYRb356xn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2c8NjXIDgk
++ cat /tmp/tmp.EWYRb356xn
++ rm /tmp/tmp.2c8NjXIDgk /tmp/tmp.EWYRb356xn
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 6 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ucuHWG4Oi3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VvEnOMFjfG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ucuHWG4Oi3
++ cat /tmp/tmp.VvEnOMFjfG
++ rm /tmp/tmp.ucuHWG4Oi3 /tmp/tmp.VvEnOMFjfG
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UB5x6q6vJj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.czV92f2b6P
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.UB5x6q6vJj
++ cat /tmp/tmp.czV92f2b6P
++ rm /tmp/tmp.UB5x6q6vJj /tmp/tmp.czV92f2b6P
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.Z79u2ugKQl
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.Ko1Fg9Gloj
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.Z79u2ugKQl
+++++ cat /tmp/tmp.Ko1Fg9Gloj
+++++ rm /tmp/tmp.Z79u2ugKQl /tmp/tmp.Ko1Fg9Gloj
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.d2uzHQ1cvj
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.WdlnfWGdkI
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.d2uzHQ1cvj
+++++ cat /tmp/tmp.WdlnfWGdkI
+++++ rm /tmp/tmp.d2uzHQ1cvj /tmp/tmp.WdlnfWGdkI
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nSbECxACJX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Y67IBqF7ih
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nSbECxACJX
++ cat /tmp/tmp.Y67IBqF7ih
++ rm /tmp/tmp.nSbECxACJX /tmp/tmp.Y67IBqF7ih
++ return 0
+ [[ 2 == \2 ]]
+ compare_mysql_cmd_local select-3 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 '' pxc
+ local command_id=select-3
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\'''
+ local pod=some-name-pxc-0
+ local postfix=
+ local container_name=pxc
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql_local 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 pxc
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\'''
+ local pod=some-name-pxc-0
+ local container_name=pxc
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-3.sql /tmp/tmp.x1AMCGp4R5/select-3.sql
+ desc 'test monitor'
+ set +o xtrace


-----------------------------------------------------------------------------------
test monitor
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=monitor
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.sYP6nzYRbr
++ mktemp
+ local LAST_ERR=/tmp/tmp.1vdHHlks5y
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.sYP6nzYRbr
secret/my-cluster-secrets patched
+ cat /tmp/tmp.1vdHHlks5y
+ rm /tmp/tmp.sYP6nzYRbr /tmp/tmp.1vdHHlks5y
+ return 0
+ wait_for_password_propagation my-cluster-secrets monitor
+ local secret=my-cluster-secrets
+ local user=monitor
+ local max_retry=240
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LnCMMpS9Yw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EKrQ7NZ59p
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LnCMMpS9Yw
++ cat /tmp/tmp.EKrQ7NZ59p
++ rm /tmp/tmp.LnCMMpS9Yw /tmp/tmp.EKrQ7NZ59p
++ return 0
+ local root_pass=test-password
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ echo 'Skipping dual password feature doesn'\''t work for 5.7. PXC 5.7 doesn'\''t support it!'
Skipping dual password feature doesn't work for 5.7. PXC 5.7 doesn't support it!
+ return
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JgvAckYxJZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hVys417kGG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.JgvAckYxJZ
++ cat /tmp/tmp.hVys417kGG
++ rm /tmp/tmp.JgvAckYxJZ /tmp/tmp.hVys417kGG
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pjNttDdSLg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cT25dPVeho
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pjNttDdSLg
++ cat /tmp/tmp.cT25dPVeho
++ rm /tmp/tmp.pjNttDdSLg /tmp/tmp.cT25dPVeho
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VYO98tKvnK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DpHAk8soxh
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VYO98tKvnK
++ cat /tmp/tmp.DpHAk8soxh
++ rm /tmp/tmp.VYO98tKvnK /tmp/tmp.DpHAk8soxh
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BuZdKF60CW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ruhJZGvHpO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BuZdKF60CW
++ cat /tmp/tmp.ruhJZGvHpO
++ rm /tmp/tmp.BuZdKF60CW /tmp/tmp.ruhJZGvHpO
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.H9LTxJhFfZ
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.7jadDPHVAq
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.H9LTxJhFfZ
+++++ cat /tmp/tmp.7jadDPHVAq
+++++ rm /tmp/tmp.H9LTxJhFfZ /tmp/tmp.7jadDPHVAq
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.iNbIbYkdeJ
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.y1k9E3ybJi
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.iNbIbYkdeJ
+++++ cat /tmp/tmp.y1k9E3ybJi
+++++ rm /tmp/tmp.iNbIbYkdeJ /tmp/tmp.y1k9E3ybJi
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eSHMkG6QBe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FbdB01XFch
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eSHMkG6QBe
++ cat /tmp/tmp.FbdB01XFch
++ rm /tmp/tmp.eSHMkG6QBe /tmp/tmp.FbdB01XFch
++ return 0
+ [[ 2 == \2 ]]
+ sleep 10
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.twrbCMUKZO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9jeUJHw4kR
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.twrbCMUKZO
++ cat /tmp/tmp.9jeUJHw4kR
++ rm /tmp/tmp.twrbCMUKZO /tmp/tmp.9jeUJHw4kR
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b7f8bcff9-gg4pr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-4.sql /tmp/tmp.x1AMCGp4R5/select-4.sql
+ desc 'test operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
test operator
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets operator dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=operator
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.yJCgu7R1uF
++ mktemp
+ local LAST_ERR=/tmp/tmp.1UAfrhPvQq
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.yJCgu7R1uF
secret/my-cluster-secrets patched
+ cat /tmp/tmp.1UAfrhPvQq
+ rm /tmp/tmp.yJCgu7R1uF /tmp/tmp.1UAfrhPvQq
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.svyA4bdEdy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.umqed5t0pg
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.svyA4bdEdy
++ cat /tmp/tmp.umqed5t0pg
++ rm /tmp/tmp.svyA4bdEdy /tmp/tmp.umqed5t0pg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tNwawsGB96
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZTe0nOKKZO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tNwawsGB96
++ cat /tmp/tmp.ZTe0nOKKZO
++ rm /tmp/tmp.tNwawsGB96 /tmp/tmp.ZTe0nOKKZO
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5Zu8MVqkr5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dL8GVbC55p
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.5Zu8MVqkr5
++ cat /tmp/tmp.dL8GVbC55p
++ rm /tmp/tmp.5Zu8MVqkr5 /tmp/tmp.dL8GVbC55p
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.rDZID2YGiJ
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.s4WxC9VXpg
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.rDZID2YGiJ
+++++ cat /tmp/tmp.s4WxC9VXpg
+++++ rm /tmp/tmp.rDZID2YGiJ /tmp/tmp.s4WxC9VXpg
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.5fLvulPPE8
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.1guer7WaOe
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.5fLvulPPE8
+++++ cat /tmp/tmp.1guer7WaOe
+++++ rm /tmp/tmp.5fLvulPPE8 /tmp/tmp.1guer7WaOe
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6JUvqjxf3L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tl1GAUsxiE
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6JUvqjxf3L
++ cat /tmp/tmp.tl1GAUsxiE
++ rm /tmp/tmp.6JUvqjxf3L /tmp/tmp.tl1GAUsxiE
++ return 0
+ [[ 2 == \2 ]]
+ sleep 10
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XeeUq4rxZc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z3KZtRURSz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XeeUq4rxZc
++ cat /tmp/tmp.Z3KZtRURSz
++ rm /tmp/tmp.XeeUq4rxZc /tmp/tmp.Z3KZtRURSz
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b7f8bcff9-gg4pr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-4.sql /tmp/tmp.x1AMCGp4R5/select-4.sql
+ desc 'change secret name'
+ set +o xtrace


-----------------------------------------------------------------------------------
change secret name
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.7hMAZWHrfX
++ mktemp
+ local LAST_ERR=/tmp/tmp.XKIGyrZTtc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7hMAZWHrfX
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.XKIGyrZTtc
+ rm /tmp/tmp.7hMAZWHrfX /tmp/tmp.XKIGyrZTtc
+ return 0
+ sleep 30
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QLdwjBdogv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.02AxqKWN3z
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.QLdwjBdogv
++ cat /tmp/tmp.02AxqKWN3z
++ rm /tmp/tmp.QLdwjBdogv /tmp/tmp.02AxqKWN3z
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Bw5ITzVDQL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nccaJIsaRs
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Bw5ITzVDQL
++ cat /tmp/tmp.nccaJIsaRs
++ rm /tmp/tmp.Bw5ITzVDQL /tmp/tmp.nccaJIsaRs
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lJH5cFNCJ4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rZaxNp0PTi
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.lJH5cFNCJ4
++ cat /tmp/tmp.rZaxNp0PTi
++ rm /tmp/tmp.lJH5cFNCJ4 /tmp/tmp.rZaxNp0PTi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IuW482N2qO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nogHoYXPua
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IuW482N2qO
++ cat /tmp/tmp.nogHoYXPua
++ rm /tmp/tmp.IuW482N2qO /tmp/tmp.nogHoYXPua
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D4j2bMsSQE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ECPQDbkTx2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.D4j2bMsSQE
++ cat /tmp/tmp.ECPQDbkTx2
++ rm /tmp/tmp.D4j2bMsSQE /tmp/tmp.ECPQDbkTx2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5CCbGJM1KY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IXpgfR5qql
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.5CCbGJM1KY
++ cat /tmp/tmp.IXpgfR5qql
++ rm /tmp/tmp.5CCbGJM1KY /tmp/tmp.IXpgfR5qql
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.04aGreZAeV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Kwfn37AbuD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.04aGreZAeV
++ cat /tmp/tmp.Kwfn37AbuD
++ rm /tmp/tmp.04aGreZAeV /tmp/tmp.Kwfn37AbuD
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XIvcTvKmRR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qbIgv9RX7z
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XIvcTvKmRR
++ cat /tmp/tmp.qbIgv9RX7z
++ rm /tmp/tmp.XIvcTvKmRR /tmp/tmp.qbIgv9RX7z
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.VspuKs2xl0
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.ULcj6GG5VY
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.VspuKs2xl0
+++++ cat /tmp/tmp.ULcj6GG5VY
+++++ rm /tmp/tmp.VspuKs2xl0 /tmp/tmp.ULcj6GG5VY
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.M9To1ROP8Z
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.SvPZBncvIi
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.M9To1ROP8Z
+++++ cat /tmp/tmp.SvPZBncvIi
+++++ rm /tmp/tmp.M9To1ROP8Z /tmp/tmp.SvPZBncvIi
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y5cDUOZjHd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yDSODbC4Ct
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.y5cDUOZjHd
++ cat /tmp/tmp.yDSODbC4Ct
++ rm /tmp/tmp.y5cDUOZjHd /tmp/tmp.yDSODbC4Ct
++ return 0
+ [[ 2 == \2 ]]
+ desc 'test new operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
test new operator
-----------------------------------------------------------------------------------

+ newpass=test-password2
++ echo -n test-password2
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZDI=
+ patch_secret my-cluster-secrets-2 operator dGVzdC1wYXNzd29yZDI=
+ local secret=my-cluster-secrets-2
+ local key=operator
+ local value=dGVzdC1wYXNzd29yZDI=
+ kubectl_bin patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Y3euac6U2v
++ mktemp
+ local LAST_ERR=/tmp/tmp.ocp31FRNAB
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Y3euac6U2v
secret/my-cluster-secrets-2 patched
+ cat /tmp/tmp.ocp31FRNAB
+ rm /tmp/tmp.Y3euac6U2v /tmp/tmp.ocp31FRNAB
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ae6yYubnIQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2SOJB6ruEO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ae6yYubnIQ
++ cat /tmp/tmp.2SOJB6ruEO
++ rm /tmp/tmp.ae6yYubnIQ /tmp/tmp.2SOJB6ruEO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AfyMHRASjl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sieJ72Ng8t
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AfyMHRASjl
++ cat /tmp/tmp.sieJ72Ng8t
++ rm /tmp/tmp.AfyMHRASjl /tmp/tmp.sieJ72Ng8t
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2pQ4drz8DM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fjm9zf1R7U
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2pQ4drz8DM
++ cat /tmp/tmp.fjm9zf1R7U
++ rm /tmp/tmp.2pQ4drz8DM /tmp/tmp.fjm9zf1R7U
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.hpbLSs8OTX
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.7kBCStr2Ne
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.hpbLSs8OTX
+++++ cat /tmp/tmp.7kBCStr2Ne
+++++ rm /tmp/tmp.hpbLSs8OTX /tmp/tmp.7kBCStr2Ne
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.EtKUdrMV6Y
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.fwm92xrKjq
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.EtKUdrMV6Y
+++++ cat /tmp/tmp.fwm92xrKjq
+++++ rm /tmp/tmp.EtKUdrMV6Y /tmp/tmp.fwm92xrKjq
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Pw2YgrOrGc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mif7wELgYd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Pw2YgrOrGc
++ cat /tmp/tmp.mif7wELgYd
++ rm /tmp/tmp.Pw2YgrOrGc /tmp/tmp.mif7wELgYd
++ return 0
+ [[ 2 == \2 ]]
+ sleep 20
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JNiHeLTTQM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yTMERVzhvd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.JNiHeLTTQM
++ cat /tmp/tmp.yTMERVzhvd
++ rm /tmp/tmp.JNiHeLTTQM /tmp/tmp.yTMERVzhvd
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b7f8bcff9-gg4pr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-4.sql /tmp/tmp.x1AMCGp4R5/select-4.sql
++ getSecretData my-cluster-secrets-2 root
++ local secretName=my-cluster-secrets-2
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets-2 '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cHiPIyjzA1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NCWwe7b0hq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets-2 '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cHiPIyjzA1
++ cat /tmp/tmp.NCWwe7b0hq
++ rm /tmp/tmp.cHiPIyjzA1 /tmp/tmp.NCWwe7b0hq
++ return 0
+ newpass=')jj<zHb^=]ReSF)q'
+ desc 'test new users sync'
+ set +o xtrace


-----------------------------------------------------------------------------------
test new users sync
-----------------------------------------------------------------------------------

+ run_mysql 'CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\'')jj<zHb^=]ReSF)q'\'';' '-h some-name-pxc -uroot -p'\'')jj<zHb^=]ReSF)q'\'''
+ local 'command=CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\'')jj<zHb^=]ReSF)q'\'';'
+ local 'uri=-h some-name-pxc -uroot -p'\'')jj<zHb^=]ReSF)q'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UGEyQ1mfLi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.u4P2VZGZeX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.UGEyQ1mfLi
++ cat /tmp/tmp.u4P2VZGZeX
++ rm /tmp/tmp.UGEyQ1mfLi /tmp/tmp.u4P2VZGZeX
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b7f8bcff9-gg4pr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ sleep 40
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\'')jj<zHb^=]ReSF)q'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -utestsync -p'\'')jj<zHb^=]ReSF)q'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\'')jj<zHb^=]ReSF)q'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -utestsync -p'\'')jj<zHb^=]ReSF)q'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.48k2Uklmho
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KrgGRHvkMW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.48k2Uklmho
++ cat /tmp/tmp.KrgGRHvkMW
++ rm /tmp/tmp.48k2Uklmho /tmp/tmp.KrgGRHvkMW
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b7f8bcff9-gg4pr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-4.sql /tmp/tmp.x1AMCGp4R5/select-4.sql
++ getSecretData internal-some-name operator
++ local secretName=internal-some-name
++ local dataKey=operator
++ kubectl_bin get secrets/internal-some-name '--template={{.data.operator}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pcHlVCDtyt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hQRZ8IXk3G
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/internal-some-name '--template={{.data.operator}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pcHlVCDtyt
++ cat /tmp/tmp.hQRZ8IXk3G
++ rm /tmp/tmp.pcHlVCDtyt /tmp/tmp.hQRZ8IXk3G
++ return 0
+ pass=test-password2
+ desc 'check secret without operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
check secret without operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.9woLsQlY4V
++ mktemp
+ local LAST_ERR=/tmp/tmp.1ua5FHtMFc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.9woLsQlY4V
secret/my-cluster-secrets-2 configured
+ cat /tmp/tmp.1ua5FHtMFc
Warning: resource secrets/my-cluster-secrets-2 is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.9woLsQlY4V /tmp/tmp.1ua5FHtMFc
+ return 0
+ sleep 15
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pv2mhkiefA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k8YfmZABRS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pv2mhkiefA
++ cat /tmp/tmp.k8YfmZABRS
++ rm /tmp/tmp.pv2mhkiefA /tmp/tmp.k8YfmZABRS
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b7f8bcff9-gg4pr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-4.sql /tmp/tmp.x1AMCGp4R5/select-4.sql
+ newpass=test-password2
++ echo -n test-password2
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZDI=
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/conf/some-name.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/conf/some-name.yml
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/conf/some-name.yml
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1753-50eda1a3#'
+ local LAST_OUT=/tmp/tmp.XWOQKtFuJQ
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ local LAST_ERR=/tmp/tmp.ioii21H7Xc
+ local exit_status=0
++ seq 0 2
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-16820~
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XWOQKtFuJQ
perconaxtradbcluster.pxc.percona.com/some-name configured
+ cat /tmp/tmp.ioii21H7Xc
+ rm /tmp/tmp.XWOQKtFuJQ /tmp/tmp.ioii21H7Xc
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0CNVlZgwfM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fsJU7mw94I
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0CNVlZgwfM
++ cat /tmp/tmp.fsJU7mw94I
++ rm /tmp/tmp.0CNVlZgwfM /tmp/tmp.fsJU7mw94I
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hIy0Yq4wAd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YO38eO77ez
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hIy0Yq4wAd
++ cat /tmp/tmp.YO38eO77ez
++ rm /tmp/tmp.hIy0Yq4wAd /tmp/tmp.YO38eO77ez
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NJ8WjJE1ll
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yPjc1xfX9s
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NJ8WjJE1ll
++ cat /tmp/tmp.yPjc1xfX9s
++ rm /tmp/tmp.NJ8WjJE1ll /tmp/tmp.yPjc1xfX9s
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2NeUjaqSz2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A01WEvfEvO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2NeUjaqSz2
++ cat /tmp/tmp.A01WEvfEvO
++ rm /tmp/tmp.2NeUjaqSz2 /tmp/tmp.A01WEvfEvO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vigsxbwaIn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CceKpAaYwb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vigsxbwaIn
++ cat /tmp/tmp.CceKpAaYwb
++ rm /tmp/tmp.vigsxbwaIn /tmp/tmp.CceKpAaYwb
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XHvPNcQtck
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0Q1iBkqgL9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XHvPNcQtck
++ cat /tmp/tmp.0Q1iBkqgL9
++ rm /tmp/tmp.XHvPNcQtck /tmp/tmp.0Q1iBkqgL9
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cIpffky3cT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.437Ib1Mt3M
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cIpffky3cT
++ cat /tmp/tmp.437Ib1Mt3M
++ rm /tmp/tmp.cIpffky3cT /tmp/tmp.437Ib1Mt3M
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 6 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EevNFx0p9N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PIVSSC3aHA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.EevNFx0p9N
++ cat /tmp/tmp.PIVSSC3aHA
++ rm /tmp/tmp.EevNFx0p9N /tmp/tmp.PIVSSC3aHA
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 7 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uAojA1T4J9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.do8wtTba8l
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.uAojA1T4J9
++ cat /tmp/tmp.do8wtTba8l
++ rm /tmp/tmp.uAojA1T4J9 /tmp/tmp.do8wtTba8l
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 8 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u4lBIg9vDD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZOFeWI4tcT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.u4lBIg9vDD
++ cat /tmp/tmp.ZOFeWI4tcT
++ rm /tmp/tmp.u4lBIg9vDD /tmp/tmp.ZOFeWI4tcT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 9 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IhABlOJ9uk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.blgZOQiSZo
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IhABlOJ9uk
++ cat /tmp/tmp.blgZOQiSZo
++ rm /tmp/tmp.IhABlOJ9uk /tmp/tmp.blgZOQiSZo
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 10 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2KOlJRHa4P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GSTSDdLNuj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2KOlJRHa4P
++ cat /tmp/tmp.GSTSDdLNuj
++ rm /tmp/tmp.2KOlJRHa4P /tmp/tmp.GSTSDdLNuj
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 11 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nwzcNbjpsR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jsBTATz9oP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nwzcNbjpsR
++ cat /tmp/tmp.jsBTATz9oP
++ rm /tmp/tmp.nwzcNbjpsR /tmp/tmp.jsBTATz9oP
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 12 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QRB1Pb1lQd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.REdU7z7m0V
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.QRB1Pb1lQd
++ cat /tmp/tmp.REdU7z7m0V
++ rm /tmp/tmp.QRB1Pb1lQd /tmp/tmp.REdU7z7m0V
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P2FOOTsbg9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cfvW7LrLxy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.P2FOOTsbg9
++ cat /tmp/tmp.cfvW7LrLxy
++ rm /tmp/tmp.P2FOOTsbg9 /tmp/tmp.cfvW7LrLxy
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.Y7bbgeU61f
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.lNeK3RDUTc
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.Y7bbgeU61f
+++++ cat /tmp/tmp.lNeK3RDUTc
+++++ rm /tmp/tmp.Y7bbgeU61f /tmp/tmp.lNeK3RDUTc
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PffLM5HWWZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8ZM5lrlArC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PffLM5HWWZ
++ cat /tmp/tmp.8ZM5lrlArC
++ rm /tmp/tmp.PffLM5HWWZ /tmp/tmp.8ZM5lrlArC
++ return 0
+ [[ 3 == \3 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ check_generation 2 haproxy some-name
+ local generation=2
+ local container=haproxy
+ local cluster=some-name
+ local current_generation
++ kubectl_bin get statefulset some-name-haproxy -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ebHF3Pto2F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5v3w3ZuAm1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-haproxy -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ebHF3Pto2F
++ cat /tmp/tmp.5v3w3ZuAm1
++ rm /tmp/tmp.ebHF3Pto2F /tmp/tmp.5v3w3ZuAm1
++ return 0
+ current_generation=2
+ [[ 2 != \2 ]]
+ patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZDI=
+ local secret=my-cluster-secrets
+ local key=monitor
+ local value=dGVzdC1wYXNzd29yZDI=
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.8RpzeA03TE
++ mktemp
+ local LAST_ERR=/tmp/tmp.gMsA9JvIot
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8RpzeA03TE
secret/my-cluster-secrets patched
+ cat /tmp/tmp.gMsA9JvIot
+ rm /tmp/tmp.8RpzeA03TE /tmp/tmp.gMsA9JvIot
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kuCgN7XOCS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TLZ59mzGv6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kuCgN7XOCS
++ cat /tmp/tmp.TLZ59mzGv6
++ rm /tmp/tmp.kuCgN7XOCS /tmp/tmp.TLZ59mzGv6
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X8elWotFPE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VvmIVWUuNT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.X8elWotFPE
++ cat /tmp/tmp.VvmIVWUuNT
++ rm /tmp/tmp.X8elWotFPE /tmp/tmp.VvmIVWUuNT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3yjwjaAq0g
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oVJ2r2vmM0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3yjwjaAq0g
++ cat /tmp/tmp.oVJ2r2vmM0
++ rm /tmp/tmp.3yjwjaAq0g /tmp/tmp.oVJ2r2vmM0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yAkTtai8gX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Sla7MeMPXS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.yAkTtai8gX
++ cat /tmp/tmp.Sla7MeMPXS
++ rm /tmp/tmp.yAkTtai8gX /tmp/tmp.Sla7MeMPXS
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pggMaoboUL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jnQ0DvBSci
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pggMaoboUL
++ cat /tmp/tmp.jnQ0DvBSci
++ rm /tmp/tmp.pggMaoboUL /tmp/tmp.jnQ0DvBSci
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.EZHXv4xccY
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.M62AyzNQg9
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.EZHXv4xccY
+++++ cat /tmp/tmp.M62AyzNQg9
+++++ rm /tmp/tmp.EZHXv4xccY /tmp/tmp.M62AyzNQg9
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9p02n9OXz7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qN1WmLzJY0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9p02n9OXz7
++ cat /tmp/tmp.qN1WmLzJY0
++ rm /tmp/tmp.9p02n9OXz7 /tmp/tmp.qN1WmLzJY0
++ return 0
+ [[ 3 == \3 ]]
+ compare_mysql_cmd select-3 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local command_id=select-3
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zan7DbTFIS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FjEzKofBz7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zan7DbTFIS
++ cat /tmp/tmp.FjEzKofBz7
++ rm /tmp/tmp.zan7DbTFIS /tmp/tmp.FjEzKofBz7
++ return 0
+ client_pod=pxc-client-7b7f8bcff9-gg4pr
+ wait_pod pxc-client-7b7f8bcff9-gg4pr
+ local pod=pxc-client-7b7f8bcff9-gg4pr
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b7f8bcff9-gg4pr
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b7f8bcff9-gg4pr condition met
pxc-client-7b7f8bcff9-gg4pr.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.x1AMCGp4R5/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1753/e2e-tests/users/compare/select-3.sql /tmp/tmp.x1AMCGp4R5/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ check_generation 3 haproxy some-name
+ local generation=3
+ local container=haproxy
+ local cluster=some-name
+ local current_generation
++ kubectl_bin get statefulset some-name-haproxy -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sEw77kZkM9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pQ2npdYHfo
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-haproxy -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sEw77kZkM9
++ cat /tmp/tmp.pQ2npdYHfo
++ rm /tmp/tmp.sEw77kZkM9 /tmp/tmp.pQ2npdYHfo
++ return 0
+ current_generation=3
+ [[ 3 != \3 ]]
+ destroy users-16820
+ local namespace=users-16820
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ grep -v level=info
+ grep -v 'the object has been modified'
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+ sort -u
+ tee /tmp/tmp.x1AMCGp4R5/operator.log
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ grep -v 'get backup status: Job.batch'
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GfskPvrhLC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hZR7Wbd5X6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GfskPvrhLC
++ cat /tmp/tmp.hZR7Wbd5X6
++ rm /tmp/tmp.GfskPvrhLC /tmp/tmp.hZR7Wbd5X6
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-6f69b446fd-lkmvc
++ mktemp
+ local LAST_OUT=/tmp/tmp.gCqVCBLtty
++ mktemp
+ local LAST_ERR=/tmp/tmp.ikQYeYAIaE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-6f69b446fd-lkmvc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gCqVCBLtty
+ cat /tmp/tmp.ikQYeYAIaE
+ rm /tmp/tmp.gCqVCBLtty /tmp/tmp.ikQYeYAIaE
+ return 0
2024-07-17T11:51:00.582Z	INFO	setup	Manager starting up	{"gitCommit": "50eda1a33a5de8689c5809cac616bd248a71cf6d", "gitBranch": "PR-1753-50eda1a3", "buildTime": "2024-07-17T09:55:54Z", "goVersion": "go1.22.5", "os": "linux", "arch": "amd64"}
2024-07-17T11:51:00.582Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.27.15-gke.1154000"}
2024-07-17T11:51:00.583Z	INFO	setup	Registering Components.
2024-07-17T11:51:05.957Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2024-07-17T11:51:05.973Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2024-07-17T11:51:05.973Z	INFO	controller-runtime.metrics	Starting metrics server
2024-07-17T11:51:05.973Z	INFO	controller-runtime.webhook	Starting webhook server
2024-07-17T11:51:05.973Z	INFO	setup	Starting the Cmd.
2024-07-17T11:51:05.973Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2024-07-17T11:51:05.974Z	INFO	controller-runtime.certwatcher	Starting certificate watcher
2024-07-17T11:51:05.974Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate
2024-07-17T11:51:05.974Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2024-07-17T11:51:06.075Z	INFO	attempting to acquire leader lease pxc-operator/08db1feb.percona.com...
2024-07-17T11:51:06.170Z	DEBUG	events	percona-xtradb-cluster-operator-6f69b446fd-lkmvc_d940a616-1bad-44eb-b1c3-acfc2574a3eb became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"bffd1f74-c5ae-4638-9f2f-1d1c5fcda4fb","apiVersion":"coordination.k8s.io/v1","resourceVersion":"68639"}, "reason": "LeaderElection"}
2024-07-17T11:51:06.170Z	INFO	Starting Controller	{"controller": "pxcbackup-controller"}
2024-07-17T11:51:06.170Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2024-07-17T11:51:06.170Z	INFO	Starting Controller	{"controller": "pxcrestore-controller"}
2024-07-17T11:51:06.170Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2024-07-17T11:51:06.170Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2024-07-17T11:51:06.170Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2024-07-17T11:51:06.170Z	INFO	successfully acquired lease pxc-operator/08db1feb.percona.com
2024-07-17T11:51:06.284Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2024-07-17T11:51:06.284Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "worker count": 1}
2024-07-17T11:51:06.285Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "worker count": 1}
2024-07-17T11:51:36.241Z	INFO	Set CR version	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "334926f7-813c-4e1a-bcb3-5aefdd88ace5", "version": "1.15.0"}
2024-07-17T11:51:36.709Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "334926f7-813c-4e1a-bcb3-5aefdd88ace5", "object": "some-name-pxc"}
2024-07-17T11:51:36.858Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "334926f7-813c-4e1a-bcb3-5aefdd88ace5", "object": "some-name-pxc"}
2024-07-17T11:51:37.060Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "334926f7-813c-4e1a-bcb3-5aefdd88ace5", "object": "some-name-pxc-unready"}
2024-07-17T11:51:37.379Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "334926f7-813c-4e1a-bcb3-5aefdd88ace5", "object": "some-name-proxysql"}
2024-07-17T11:51:37.491Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "334926f7-813c-4e1a-bcb3-5aefdd88ace5", "object": "some-name-proxysql"}
2024-07-17T11:51:37.688Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "334926f7-813c-4e1a-bcb3-5aefdd88ace5", "object": "some-name-proxysql-unready"}
2024-07-17T11:51:38.715Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "06382c12-e8ba-48bb-95e8-36c3bb677e4b", "object": "some-name-pxc"}
2024-07-17T11:51:38.945Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "06382c12-e8ba-48bb-95e8-36c3bb677e4b", "object": "some-name-proxysql"}
2024-07-17T11:52:51.116Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "a32ff2b0-a772-428d-8a97-13fa0b87296a", "user": "operator"}
2024-07-17T11:52:51.151Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "a32ff2b0-a772-428d-8a97-13fa0b87296a", "user": "monitor"}
2024-07-17T11:52:51.205Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "a32ff2b0-a772-428d-8a97-13fa0b87296a"}
2024-07-17T11:52:51.245Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "a32ff2b0-a772-428d-8a97-13fa0b87296a", "user": "xtrabackup"}
2024-07-17T11:52:51.290Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "a32ff2b0-a772-428d-8a97-13fa0b87296a"}
2024-07-17T11:52:51.385Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "a32ff2b0-a772-428d-8a97-13fa0b87296a", "err": "get primary pxc pod: not found"}
2024-07-17T11:52:56.139Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "5cd945cc-9948-4881-8ea5-59136f7882c5", "err": "get primary pxc pod: not found"}
2024-07-17T11:53:01.301Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "045e1099-48ef-4f08-bdee-7b04c5ab8ea6", "err": "get primary pxc pod: not found"}
2024-07-17T11:53:06.460Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "50a24a1e-7b99-417c-8fb5-08613d9d57b9", "err": "get primary pxc pod: not found"}
2024-07-17T11:55:17.265Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "bf9174d9-3dcf-4cc1-9489-8517a3d17f39", "user": "root"}
2024-07-17T11:55:17.307Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "bf9174d9-3dcf-4cc1-9489-8517a3d17f39", "user": "replication"}
2024-07-17T11:55:17.462Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "bf9174d9-3dcf-4cc1-9489-8517a3d17f39", "new version": "5.7.44-48-57"}
2024-07-17T11:55:20.819Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "bf9174d9-3dcf-4cc1-9489-8517a3d17f39"}
2024-07-17T11:55:25.716Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "42d45f8f-0aab-4898-9eac-869dbc062117"}
2024-07-17T11:55:30.790Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "2f340875-af99-4b82-829c-a95d1cde4e13"}
2024-07-17T11:55:35.903Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "e35a1ab6-4967-4d14-a0aa-455062ceeb2c"}
2024-07-17T11:55:42.095Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "25e7b1c3-b854-4fbf-bcae-1642ac681189"}
2024-07-17T11:55:47.694Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "427b9902-d347-4327-94c7-c699adb536b2"}
2024-07-17T11:55:52.720Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "e50c41ef-ff42-48f1-af6a-76fd066d5858"}
2024-07-17T11:55:58.092Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "50013f62-7233-4758-aa16-53027fe56f69"}
2024-07-17T11:56:03.400Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "05b16fa7-9052-4e36-a8d1-6c0910c1f0f9"}
2024-07-17T11:56:08.351Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "fc08e142-de0a-4a2c-a8df-c0b76fe72030"}
2024-07-17T11:56:13.404Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "221392a9-8752-44dc-a43a-251a705171c1"}
2024-07-17T11:56:18.711Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "3bd91d71-4593-45cf-87da-badb0201e5f6"}
2024-07-17T11:56:23.906Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "c2bd8a6c-eabf-4193-b987-f800336ec125"}
2024-07-17T11:56:29.233Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "79297788-1f8a-46a0-b8ba-b6e15a70476c"}
2024-07-17T11:56:31.028Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b9fa4a50-3a42-4139-b339-6b721118fb10", "user": "root"}
2024-07-17T11:56:31.068Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b9fa4a50-3a42-4139-b339-6b721118fb10", "user": "root"}
2024-07-17T11:56:31.078Z	INFO	MySQL init secret created	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b9fa4a50-3a42-4139-b339-6b721118fb10", "secret": "some-name-mysql-init", "user": "root"}
2024-07-17T11:56:37.015Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b9fa4a50-3a42-4139-b339-6b721118fb10"}
2024-07-17T11:56:37.031Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b9fa4a50-3a42-4139-b339-6b721118fb10", "user": "root"}
2024-07-17T11:56:40.511Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b9fa4a50-3a42-4139-b339-6b721118fb10"}
2024-07-17T11:56:45.521Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "7a055d99-4012-40f2-a37f-2f9dd3373b27"}
2024-07-17T11:56:50.384Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "97d942e0-7414-494d-bdf7-110d5c2b9cf3", "object": "some-name-proxysql"}
2024-07-17T11:56:50.748Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "6eb1cd9e-6a0b-4b91-b58e-cf9feacbb3ab"}
2024-07-17T11:57:18.558Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "ac830905-29e9-4249-abb5-83713f08131f", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:922\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1226\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-17T11:57:23.473Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "79d9708f-7c8e-42dc-b8ab-1f7db06131b7", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:922\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1226\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-17T11:57:28.859Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "44855ce4-d954-47bd-8c9e-40a31953c427", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:922\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1226\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-17T11:57:37.539Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9a37f56e-fe14-47cd-82a4-76f824a65e19"}
2024-07-17T11:57:40.313Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "be0fb4ce-6e65-4d85-a9bb-a09920791248", "user": "proxyadmin"}
2024-07-17T11:57:40.313Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "be0fb4ce-6e65-4d85-a9bb-a09920791248", "user": "proxyadmin"}
2024-07-17T11:57:40.384Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "be0fb4ce-6e65-4d85-a9bb-a09920791248", "user": "proxyadmin"}
2024-07-17T11:57:40.395Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "be0fb4ce-6e65-4d85-a9bb-a09920791248", "user": "proxyadmin"}
2024-07-17T11:57:40.395Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "be0fb4ce-6e65-4d85-a9bb-a09920791248", "last-applied-secret": "1a55867d0f4fa0d0643653b2eb57b2f690315d8d843a10599c9839c2a779a95f"}
2024-07-17T11:57:40.430Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "be0fb4ce-6e65-4d85-a9bb-a09920791248", "object": "some-name-proxysql"}
2024-07-17T11:57:40.611Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "be0fb4ce-6e65-4d85-a9bb-a09920791248", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:922\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1226\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-17T11:58:15.585Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "cbd4d185-c2cb-4b7f-a265-7bb9d0b2e301"}
2024-07-17T11:58:25.765Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "f14acb88-0878-4970-b3f5-49def2f671b4"}
2024-07-17T11:58:27.906Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "779705fc-f084-40f8-82f4-e3a85805ca77", "object": "some-name-proxysql"}
2024-07-17T11:58:31.211Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "779705fc-f084-40f8-82f4-e3a85805ca77"}
2024-07-17T11:58:34.272Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "66a3cf9f-0c19-44fe-8b39-d20341b334fd", "user": "xtrabackup"}
2024-07-17T11:58:34.297Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "66a3cf9f-0c19-44fe-8b39-d20341b334fd", "user": "xtrabackup"}
2024-07-17T11:58:34.309Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "66a3cf9f-0c19-44fe-8b39-d20341b334fd", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-07-17T11:58:34.321Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "66a3cf9f-0c19-44fe-8b39-d20341b334fd", "user": "xtrabackup"}
2024-07-17T11:58:34.321Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "66a3cf9f-0c19-44fe-8b39-d20341b334fd", "last-applied-secret": "d11bcc70ca53ea410cd455e13d82e7225917b0861d9b677ed34c02c5f6fbaf1b"}
2024-07-17T11:58:34.353Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "66a3cf9f-0c19-44fe-8b39-d20341b334fd", "object": "some-name-pxc"}
2024-07-17T11:58:39.580Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "66a3cf9f-0c19-44fe-8b39-d20341b334fd"}
2024-07-17T12:00:09.431Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "8cefb254-ab94-4238-bfb7-5dcbf4500f9d", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-16820 on 10.187.32.10:53: no such host"}
2024-07-17T12:00:09.622Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "32767e5a-d5c6-4ccd-be97-f38a1e4a97b1", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-16820 on 10.187.32.10:53: no such host"}
2024-07-17T12:00:14.704Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "c11cf182-74ea-4da7-9dab-ddd43f6129f5", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-16820 on 10.187.32.10:53: no such host"}
2024-07-17T12:00:19.907Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "912da117-26a2-4cf0-baef-bf9695d70e8a", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-16820 on 10.187.32.10:53: no such host"}
2024-07-17T12:00:50.755Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "60126a76-f817-47d9-819e-153c3b159f68", "primary name": "some-name-pxc-0.some-name-pxc.users-16820.svc.cluster.local"}
2024-07-17T12:00:55.856Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "efadb576-b869-41f1-9c59-f1b52fa4ed9d", "primary name": "some-name-pxc-0.some-name-pxc.users-16820.svc.cluster.local"}
2024-07-17T12:01:04.526Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "91f74c65-85f3-4840-a422-66faa971518d"}
2024-07-17T12:01:10.508Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "f1010986-8d95-4463-9f63-88d0cc5b619c"}
2024-07-17T12:01:14.699Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "541b76e2-33a4-438c-93b9-beec1b8ce99f"}
2024-07-17T12:01:20.119Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "90dc7e9e-4c3d-4409-a153-e0232ef358e2"}
2024-07-17T12:01:25.440Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "4f93a1a3-3e0d-4147-8fdd-1200d1fb645d"}
2024-07-17T12:01:26.973Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "f1d3f198-44b7-47fa-b01c-a5e758900c80", "user": "monitor"}
2024-07-17T12:01:27.001Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "f1d3f198-44b7-47fa-b01c-a5e758900c80", "user": "monitor"}
2024-07-17T12:01:27.011Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "f1d3f198-44b7-47fa-b01c-a5e758900c80", "secret": "some-name-mysql-init", "user": "monitor"}
2024-07-17T12:01:27.055Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "f1d3f198-44b7-47fa-b01c-a5e758900c80", "user": "monitor"}
2024-07-17T12:01:27.091Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "f1d3f198-44b7-47fa-b01c-a5e758900c80", "user": "monitor"}
2024-07-17T12:01:27.092Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "f1d3f198-44b7-47fa-b01c-a5e758900c80", "last-applied-secret": "4e3282050745046d5044e0fcb5f2bf140d7320a1256a79c8ca2184f38f4f3098"}
2024-07-17T12:01:27.134Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "f1d3f198-44b7-47fa-b01c-a5e758900c80", "object": "some-name-proxysql"}
2024-07-17T12:01:30.139Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "f1d3f198-44b7-47fa-b01c-a5e758900c80", "error": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / ", "errorVerbose": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:922\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1226\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-17T12:02:15.785Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "54449b19-79a4-41cf-b63b-99f62dd0158e"}
2024-07-17T12:02:20.488Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "41555461-0d2f-45b7-90c8-b3c8b86a1bfb"}
2024-07-17T12:02:25.875Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "c3b921b1-22c2-4ffc-ba26-54e3e648e890"}
2024-07-17T12:02:31.057Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "a4fb199a-b676-498e-8873-a9a83ac392b4"}
2024-07-17T12:02:32.735Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b7ece2fa-b199-4169-944c-ce2f46b8a98a", "user": "operator"}
2024-07-17T12:02:32.762Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b7ece2fa-b199-4169-944c-ce2f46b8a98a", "user": "operator"}
2024-07-17T12:02:32.771Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b7ece2fa-b199-4169-944c-ce2f46b8a98a", "secret": "some-name-mysql-init", "user": "operator"}
2024-07-17T12:02:32.785Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b7ece2fa-b199-4169-944c-ce2f46b8a98a", "user": "operator"}
2024-07-17T12:02:32.785Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b7ece2fa-b199-4169-944c-ce2f46b8a98a", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-07-17T12:02:32.810Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b7ece2fa-b199-4169-944c-ce2f46b8a98a", "object": "some-name-proxysql"}
2024-07-17T12:02:34.143Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b7ece2fa-b199-4169-944c-ce2f46b8a98a", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-16820.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-16820.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-16820.svc.' (using password: YES)\nERROR (line:569) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-16820.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-16820.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-16820.svc.' (using password: YES)\nERROR (line:569) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:922\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1226\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-17T12:03:07.534Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "cfa4372a-5959-4ef4-b76b-89ded7eedff2"}
2024-07-17T12:03:30.940Z	INFO	Created user secrets	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "secrets": "my-cluster-secrets-2"}
2024-07-17T12:03:30.941Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "root"}
2024-07-17T12:03:30.983Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "root"}
2024-07-17T12:03:30.994Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "secret": "some-name-mysql-init", "user": "root"}
2024-07-17T12:03:32.806Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "bdbe74eb-c460-41be-a170-5a4b062ac2a3"}
2024-07-17T12:03:35.371Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585"}
2024-07-17T12:03:35.389Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "root"}
2024-07-17T12:03:35.389Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "operator"}
2024-07-17T12:03:35.416Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "operator"}
2024-07-17T12:03:35.426Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "secret": "some-name-mysql-init", "user": "operator"}
2024-07-17T12:03:35.443Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "operator"}
2024-07-17T12:03:35.443Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "monitor"}
2024-07-17T12:03:35.467Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "monitor"}
2024-07-17T12:03:35.478Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "secret": "some-name-mysql-init", "user": "monitor"}
2024-07-17T12:03:35.522Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "monitor"}
2024-07-17T12:03:35.531Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "monitor"}
2024-07-17T12:03:35.532Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "xtrabackup"}
2024-07-17T12:03:35.554Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "xtrabackup"}
2024-07-17T12:03:35.566Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-07-17T12:03:35.576Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "xtrabackup"}
2024-07-17T12:03:35.576Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "replication"}
2024-07-17T12:03:35.602Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "replication"}
2024-07-17T12:03:35.614Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "secret": "some-name-mysql-init", "user": "replication"}
2024-07-17T12:03:35.630Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "replication"}
2024-07-17T12:03:35.630Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "proxyadmin"}
2024-07-17T12:03:35.673Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "proxyadmin"}
2024-07-17T12:03:35.686Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "user": "proxyadmin"}
2024-07-17T12:03:35.686Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "last-applied-secret": "080bb4769b0147989fd6c1547cf3a3530056bea07ffc25a96ff9346dacac5cc3"}
2024-07-17T12:03:35.686Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "last-applied-secret": "080bb4769b0147989fd6c1547cf3a3530056bea07ffc25a96ff9346dacac5cc3"}
2024-07-17T12:03:35.712Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "object": "some-name-pxc"}
2024-07-17T12:03:35.757Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "object": "some-name-proxysql"}
2024-07-17T12:03:35.916Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "d1e7e9ec-fa32-462a-9988-f344317bc585", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:922\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1226\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-17T12:03:36.572Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "8fd90e76-d6ff-41e4-b08f-bd5726b4dd97", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:922\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1226\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-17T12:05:09.863Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "2f972f4b-db54-45e4-91f7-bb6c32eacd20", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-16820 on 10.187.32.10:53: no such host"}
2024-07-17T12:05:10.105Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "861f81de-1316-4921-b8d4-3749504f6f8f", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-16820 on 10.187.32.10:53: no such host"}
2024-07-17T12:05:20.268Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "3fe9cf59-0750-4021-9a17-3736de9f41d2", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-16820 on 10.187.32.10:53: no such host"}
2024-07-17T12:05:25.432Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "8d02dc91-754f-4fec-87db-45daccb2432e", "primary name": "some-name-pxc-0.some-name-pxc.users-16820.svc.cluster.local"}
2024-07-17T12:05:30.571Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "e3dc35de-db74-4e6c-8d78-7a012319c174", "primary name": "some-name-pxc-0.some-name-pxc.users-16820.svc.cluster.local"}
2024-07-17T12:05:35.712Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "5e923883-427e-4e48-b072-0b76eccb092a", "primary name": "some-name-pxc-0.some-name-pxc.users-16820.svc.cluster.local"}
2024-07-17T12:05:40.825Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "205f9ad4-fa62-4d9d-8b8f-40b9920e84d8", "primary name": "some-name-pxc-0.some-name-pxc.users-16820.svc.cluster.local"}
2024-07-17T12:05:45.946Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "52f635d9-dde7-4ad6-a805-3ba650669c7e", "primary name": "some-name-pxc-0.some-name-pxc.users-16820.svc.cluster.local"}
2024-07-17T12:05:51.063Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "0ab9d11b-aeb9-404e-8e08-abea44b848fa", "primary name": "some-name-pxc-0.some-name-pxc.users-16820.svc.cluster.local"}
2024-07-17T12:05:59.900Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "559872e4-4a28-454f-bf80-153b335bcf3f"}
2024-07-17T12:06:04.984Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "7e7d1bb6-3104-42a0-ba66-6464254e9cb8"}
2024-07-17T12:06:10.556Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "8edaad71-9de3-46d7-8d18-d942c69202ca"}
2024-07-17T12:06:15.462Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "de00adcf-0e9a-4a37-9862-eab0f1c15138"}
2024-07-17T12:06:17.200Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "2d1b7a31-ee3d-48f4-a26f-deba50100780", "user": "operator"}
2024-07-17T12:06:17.229Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "2d1b7a31-ee3d-48f4-a26f-deba50100780", "user": "operator"}
2024-07-17T12:06:17.239Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "2d1b7a31-ee3d-48f4-a26f-deba50100780", "secret": "some-name-mysql-init", "user": "operator"}
2024-07-17T12:06:17.249Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "2d1b7a31-ee3d-48f4-a26f-deba50100780", "user": "operator"}
2024-07-17T12:06:17.249Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "2d1b7a31-ee3d-48f4-a26f-deba50100780", "last-applied-secret": "1b2e93800cc4766c9430158aa9a8724238c2aeb338004262389f02b39cfc2fb6"}
2024-07-17T12:06:17.276Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "2d1b7a31-ee3d-48f4-a26f-deba50100780", "object": "some-name-proxysql"}
2024-07-17T12:06:18.619Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "2d1b7a31-ee3d-48f4-a26f-deba50100780", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-16820.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-16820.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-16820.svc.' (using password: YES)\nERROR (line:569) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-16820.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-16820.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-16820.svc.' (using password: YES)\nERROR (line:569) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:922\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1226\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-17T12:06:52.610Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "3a404ffc-387d-4809-b449-8ca402ccef0c"}
2024-07-17T12:07:00.598Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "245fbee6-4010-43c5-8b2f-28451062e2dd"}
2024-07-17T12:07:05.704Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "1fd80e47-b137-4136-8016-192d58ef0d63"}
2024-07-17T12:07:10.974Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "0d1b91a0-bb5d-4abf-8cb3-5158121ecf3a"}
2024-07-17T12:07:16.073Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "bc043f8c-5afb-4e31-b57b-adf6ddc47765"}
2024-07-17T12:07:21.473Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "aaae41e2-9d52-4fb0-a12e-8ff9f5888d38"}
2024-07-17T12:07:26.613Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "e29fbb19-f07f-44df-954b-74f75159f0fe"}
2024-07-17T12:07:32.820Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "3a1264c7-dac8-4eb6-a1ae-5e34a84e9279"}
2024-07-17T12:07:38.214Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "97043211-c4da-475a-bae7-39b73601d830"}
2024-07-17T12:07:42.365Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "ddda3ca3-8e7c-4d01-9a65-5c5a744810f3"}
2024-07-17T12:07:47.804Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "757ee865-1fcf-464f-b4ed-13e5765c5d98"}
2024-07-17T12:07:52.769Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "0826fb2d-df51-46e6-9bd3-cea5331a1435"}
2024-07-17T12:07:58.003Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "c4825a02-9001-485b-9b06-754a26d6e813"}
2024-07-17T12:08:03.253Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "f66d7385-12d8-462f-865c-c0e301649e25"}
2024-07-17T12:08:08.573Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "358772cd-15e3-479b-95fb-5df3135093f9"}
2024-07-17T12:08:13.765Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "285f50ed-ec6b-40b3-9a08-c8001ebfe477"}
2024-07-17T12:08:18.977Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "2bde73d9-4668-4213-87b3-c25e74d82f30"}
2024-07-17T12:08:20.679Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "root"}
2024-07-17T12:08:20.717Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "root"}
2024-07-17T12:08:20.732Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "secret": "some-name-mysql-init", "user": "root"}
2024-07-17T12:08:26.278Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec"}
2024-07-17T12:08:26.293Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "root"}
2024-07-17T12:08:26.293Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "monitor"}
2024-07-17T12:08:26.320Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "monitor"}
2024-07-17T12:08:26.334Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "secret": "some-name-mysql-init", "user": "monitor"}
2024-07-17T12:08:26.378Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "monitor"}
2024-07-17T12:08:26.396Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "monitor"}
2024-07-17T12:08:26.396Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "xtrabackup"}
2024-07-17T12:08:26.419Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "xtrabackup"}
2024-07-17T12:08:26.435Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-07-17T12:08:26.450Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "xtrabackup"}
2024-07-17T12:08:26.451Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "proxyadmin"}
2024-07-17T12:08:26.497Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "proxyadmin"}
2024-07-17T12:08:26.517Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "user": "proxyadmin"}
2024-07-17T12:08:26.517Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "last-applied-secret": "3a481877261c792f627caf1351e638e44ef762a0ff942b9a6b74bbc577eb1b73"}
2024-07-17T12:08:26.517Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "last-applied-secret": "3a481877261c792f627caf1351e638e44ef762a0ff942b9a6b74bbc577eb1b73"}
2024-07-17T12:08:26.565Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "object": "some-name-pxc"}
2024-07-17T12:08:26.616Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "object": "some-name-proxysql"}
2024-07-17T12:08:26.740Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "35d78bb5-b848-4e43-87be-9f6fbd9d43ec", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:922\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1226\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-17T12:08:41.792Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "79eab914-b51f-4c98-af61-48e8ab7a3702", "object": "some-name-pxc"}
2024-07-17T12:08:41.930Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "79eab914-b51f-4c98-af61-48e8ab7a3702", "object": "some-name-haproxy"}
2024-07-17T12:08:41.957Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "79eab914-b51f-4c98-af61-48e8ab7a3702", "object": "some-name-haproxy"}
2024-07-17T12:08:41.999Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "79eab914-b51f-4c98-af61-48e8ab7a3702", "object": "some-name-haproxy"}
2024-07-17T12:08:42.116Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "79eab914-b51f-4c98-af61-48e8ab7a3702", "object": "some-name-haproxy-replicas"}
2024-07-17T12:08:42.532Z	INFO	KubeAPIWarningLogger	.metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: f8f2d28e-748d-4ae1-a2b1-e8954c551643
2024-07-17T12:10:56.585Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "root"}
2024-07-17T12:10:56.622Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "root"}
2024-07-17T12:10:56.636Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "secret": "some-name-mysql-init", "user": "root"}
2024-07-17T12:10:56.657Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "root"}
2024-07-17T12:10:56.657Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "operator"}
2024-07-17T12:10:56.682Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "operator"}
2024-07-17T12:10:56.693Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "secret": "some-name-mysql-init", "user": "operator"}
2024-07-17T12:10:56.704Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "operator"}
2024-07-17T12:10:56.704Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "monitor"}
2024-07-17T12:10:56.728Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "monitor"}
2024-07-17T12:10:56.741Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "secret": "some-name-mysql-init", "user": "monitor"}
2024-07-17T12:10:56.752Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "monitor"}
2024-07-17T12:10:56.752Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "xtrabackup"}
2024-07-17T12:10:56.774Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "xtrabackup"}
2024-07-17T12:10:56.789Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-07-17T12:10:56.799Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "xtrabackup"}
2024-07-17T12:10:56.799Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "replication"}
2024-07-17T12:10:56.822Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "replication"}
2024-07-17T12:10:56.833Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "secret": "some-name-mysql-init", "user": "replication"}
2024-07-17T12:10:56.847Z	INFO	HAProxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-07-17T12:10:56.847Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "user": "replication"}
2024-07-17T12:10:56.847Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-07-17T12:10:56.872Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "object": "some-name-pxc"}
2024-07-17T12:10:56.908Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "9c6f4a8e-fedb-4765-be0d-4317030087a1", "object": "some-name-haproxy"}
2024-07-17T12:13:43.730Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b497e42b-dca3-4c90-ae8f-fe1a4a0fcdd3", "user": "monitor"}
2024-07-17T12:13:43.762Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b497e42b-dca3-4c90-ae8f-fe1a4a0fcdd3", "user": "monitor"}
2024-07-17T12:13:43.776Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b497e42b-dca3-4c90-ae8f-fe1a4a0fcdd3", "secret": "some-name-mysql-init", "user": "monitor"}
2024-07-17T12:13:43.795Z	INFO	HAProxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b497e42b-dca3-4c90-ae8f-fe1a4a0fcdd3", "last-applied-secret": "81935ecf30ae523a6ee69582354f9488d47dbeb0f8c7c572af96870f2024284d"}
2024-07-17T12:13:43.795Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b497e42b-dca3-4c90-ae8f-fe1a4a0fcdd3", "user": "monitor"}
2024-07-17T12:13:43.830Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-16820", "name": "some-name", "reconcileID": "b497e42b-dca3-4c90-ae8f-fe1a4a0fcdd3", "object": "some-name-haproxy"}
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1228
+ grep -v NAMESPACE
+ kubectl get pxc --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n users-16820 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.82ROVtcdHW
++ mktemp
+ local LAST_ERR=/tmp/tmp.7mqghyPw2A
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.82ROVtcdHW
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.7mqghyPw2A
+ rm /tmp/tmp.82ROVtcdHW /tmp/tmp.7mqghyPw2A
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.X0mHI0Jngq
++ mktemp
+ local LAST_ERR=/tmp/tmp.l4sWwKEIfT
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.X0mHI0Jngq
No resources found
+ cat /tmp/tmp.l4sWwKEIfT
+ rm /tmp/tmp.X0mHI0Jngq /tmp/tmp.l4sWwKEIfT
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.LtGncjco7L
++ mktemp
+ local LAST_ERR=/tmp/tmp.YdsY2FiAYg
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.LtGncjco7L
No resources found
+ cat /tmp/tmp.YdsY2FiAYg
+ rm /tmp/tmp.LtGncjco7L /tmp/tmp.YdsY2FiAYg
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.yMfQxMxJAn
++ mktemp
+ local LAST_ERR=/tmp/tmp.1STxfi8b0A
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.yMfQxMxJAn
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.1STxfi8b0A
+ rm /tmp/tmp.yMfQxMxJAn /tmp/tmp.1STxfi8b0A
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace users-16820
+ rm -rf /tmp/tmp.x1AMCGp4R5
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.wwsMXveaim
+ local LAST_OUT=/tmp/tmp.jgo0zw7cUG
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.1tyMHXZigw
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.D55Vgpzhif
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace users-16820
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator