Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/logs/users-8-0.log
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
+ create_infra users-2243
+ local ns=users-2243
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n users-19055 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.YJFSOeFhRM
++ mktemp
+ local LAST_ERR=/tmp/tmp.kxVlRFwuPD
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.YJFSOeFhRM
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.kxVlRFwuPD
+ rm /tmp/tmp.YJFSOeFhRM /tmp/tmp.kxVlRFwuPD
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.fXvHRtLpa8
++ mktemp
+ local LAST_ERR=/tmp/tmp.pE8mRAVlLi
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.fXvHRtLpa8
No resources found
+ cat /tmp/tmp.pE8mRAVlLi
+ rm /tmp/tmp.fXvHRtLpa8 /tmp/tmp.pE8mRAVlLi
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.EDoOPlOtil
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ew75dBAg1s
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EDoOPlOtil
No resources found
+ cat /tmp/tmp.Ew75dBAg1s
+ rm /tmp/tmp.EDoOPlOtil /tmp/tmp.Ew75dBAg1s
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.O2Xl4WAf9D
+ local LAST_OUT=/tmp/tmp.WKguTocZhK
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.73M4LFsrRA
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.UKH2qgt8jk
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.O2Xl4WAf9D
+ cat /tmp/tmp.73M4LFsrRA
+ rm /tmp/tmp.O2Xl4WAf9D /tmp/tmp.73M4LFsrRA
+ return 0
namespace "users-19055" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WKguTocZhK
namespace "pxc-operator" deleted
+ cat /tmp/tmp.UKH2qgt8jk
+ rm /tmp/tmp.WKguTocZhK /tmp/tmp.UKH2qgt8jk
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'namespace/pxc-operator - '
namespace/pxc-operator - + set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.eEUOGO3Kar
++ mktemp
+ local LAST_ERR=/tmp/tmp.pxW5xyueUd
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.eEUOGO3Kar
namespace/pxc-operator created
+ cat /tmp/tmp.pxW5xyueUd
+ rm /tmp/tmp.eEUOGO3Kar /tmp/tmp.pxW5xyueUd
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jOCGy2bOEu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.shzS6xRajg
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.jOCGy2bOEu
++ cat /tmp/tmp.shzS6xRajg
++ rm /tmp/tmp.jOCGy2bOEu /tmp/tmp.shzS6xRajg
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1717-958d148d-1-cluster1 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.77EyFfeVjt
++ mktemp
+ local LAST_ERR=/tmp/tmp.pKm96xkhr1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1717-958d148d-1-cluster1 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.77EyFfeVjt
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1717-958d148d-1-cluster1" modified.
+ cat /tmp/tmp.pKm96xkhr1
+ rm /tmp/tmp.77EyFfeVjt /tmp/tmp.pKm96xkhr1
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.aCDzC4o0aF
++ mktemp
+ local LAST_ERR=/tmp/tmp.tHKcv8z1B7
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.aCDzC4o0aF
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.tHKcv8z1B7
+ rm /tmp/tmp.aCDzC4o0aF /tmp/tmp.tHKcv8z1B7
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.DjNbLmONFH
++ mktemp
+ local LAST_ERR=/tmp/tmp.4s2lkYKcj7
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.DjNbLmONFH
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.4s2lkYKcj7
+ rm /tmp/tmp.DjNbLmONFH /tmp/tmp.4s2lkYKcj7
+ return 0
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/deploy/cw-operator.yaml
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1717-958d148d^'
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ kubectl_bin apply -f -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.M6maWrldYo
++ mktemp
+ local LAST_ERR=/tmp/tmp.QZz7dPdqvf
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.M6maWrldYo
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.QZz7dPdqvf
+ rm /tmp/tmp.M6maWrldYo /tmp/tmp.QZz7dPdqvf
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.cqifdEH32x
++ mktemp
+ local LAST_ERR=/tmp/tmp.BuFW5SqC9F
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.cqifdEH32x
pod/percona-xtradb-cluster-operator-76cddf6977-dz79t condition met
+ cat /tmp/tmp.BuFW5SqC9F
+ rm /tmp/tmp.cqifdEH32x /tmp/tmp.BuFW5SqC9F
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nhNU4UEuMK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.edfu63STfH
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nhNU4UEuMK
++ cat /tmp/tmp.edfu63STfH
++ rm /tmp/tmp.nhNU4UEuMK /tmp/tmp.edfu63STfH
++ return 0
+ wait_pod percona-xtradb-cluster-operator-76cddf6977-dz79t 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-76cddf6977-dz79t
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-76cddf6977-dz79t
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-76cddf6977-dz79t condition met
percona-xtradb-cluster-operator-76cddf6977-dz79t.Ok
+ sleep 3
+ create_namespace users-2243
+ local namespace=users-2243
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces users-2243'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces users-2243
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace users-2243
+ xargs kubectl delete ns
++ mktemp
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.sX9gvBmpIN
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.iiFY46Trg0
+ local exit_status=0
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.eY3ftSnV8L
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-2243
+ local LAST_ERR=/tmp/tmp.kABqewR1YH
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-2243
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.eY3ftSnV8L
+ cat /tmp/tmp.kABqewR1YH
+ rm /tmp/tmp.eY3ftSnV8L /tmp/tmp.kABqewR1YH
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-2243
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.sX9gvBmpIN
+ cat /tmp/tmp.iiFY46Trg0
Error from server (NotFound): namespaces "users-2243" not found
+ rm /tmp/tmp.sX9gvBmpIN /tmp/tmp.iiFY46Trg0
+ return 1
+ :
+ wait_for_delete namespace/users-2243
+ local res=namespace/users-2243
+ echo -n 'namespace/users-2243 - '
namespace/users-2243 - + set +o xtrace
Error from server (NotFound): namespaces "users-2243" not found
+ desc 'create namespace users-2243'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace users-2243
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace users-2243
++ mktemp
+ local LAST_OUT=/tmp/tmp.aGKVF4NCiG
++ mktemp
+ local LAST_ERR=/tmp/tmp.zgHWleUCwm
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace users-2243
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.aGKVF4NCiG
namespace/users-2243 created
+ cat /tmp/tmp.zgHWleUCwm
+ rm /tmp/tmp.aGKVF4NCiG /tmp/tmp.zgHWleUCwm
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DWDAVrc3nD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VQEtItHYLM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DWDAVrc3nD
++ cat /tmp/tmp.VQEtItHYLM
++ rm /tmp/tmp.DWDAVrc3nD /tmp/tmp.VQEtItHYLM
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1717-958d148d-1-cluster1 --namespace=users-2243
++ mktemp
+ local LAST_OUT=/tmp/tmp.NgkbtrwFy4
++ mktemp
+ local LAST_ERR=/tmp/tmp.E4uX3SV3Vq
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1717-958d148d-1-cluster1 --namespace=users-2243
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.NgkbtrwFy4
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1717-958d148d-1-cluster1" modified.
+ cat /tmp/tmp.E4uX3SV3Vq
+ rm /tmp/tmp.NgkbtrwFy4 /tmp/tmp.E4uX3SV3Vq
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.6otpr8vyne
++ mktemp
+ local LAST_ERR=/tmp/tmp.2Ex0vKIIyh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6otpr8vyne
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.2Ex0vKIIyh
+ rm /tmp/tmp.6otpr8vyne /tmp/tmp.2Ex0vKIIyh
+ return 0
+ desc 'create PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC cluster
-----------------------------------------------------------------------------------

+ newpass=test-password
++ echo -n test-password
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZA==
+ cluster=some-name
+ spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/some-name.yml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/some-name.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ciJe9XOKBW
++ mktemp
+ local LAST_ERR=/tmp/tmp.vymosyTzax
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ciJe9XOKBW
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.vymosyTzax
+ rm /tmp/tmp.ciJe9XOKBW /tmp/tmp.vymosyTzax
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/client.yml
++ mktemp
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ local LAST_OUT=/tmp/tmp.4vx0Goo1PM
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
++ mktemp
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-2243~
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1717-958d148d#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ local LAST_ERR=/tmp/tmp.HGOATg4CBB
+ local exit_status=0
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.4vx0Goo1PM
deployment.apps/pxc-client created
+ cat /tmp/tmp.HGOATg4CBB
+ rm /tmp/tmp.4vx0Goo1PM /tmp/tmp.HGOATg4CBB
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/conf/some-name.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1717-958d148d#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-2243~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ local LAST_OUT=/tmp/tmp.mXZKcRJuwr
++ mktemp
+ local LAST_ERR=/tmp/tmp.0ycHfyPtTS
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.mXZKcRJuwr
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.0ycHfyPtTS
+ rm /tmp/tmp.mXZKcRJuwr /tmp/tmp.0ycHfyPtTS
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy some-name
++ local target_cluster=some-name
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kLnQAEpil2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fZ7Q1yD2eA
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.kLnQAEpil2
+++ cat /tmp/tmp.fZ7Q1yD2eA
+++ rm /tmp/tmp.kLnQAEpil2 /tmp/tmp.fZ7Q1yD2eA
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DB83cri1jG
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Loapyr8Yuc
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.DB83cri1jG
+++ cat /tmp/tmp.Loapyr8Yuc
+++ rm /tmp/tmp.DB83cri1jG /tmp/tmp.Loapyr8Yuc
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo some-name-proxysql
++ return
+ local proxy=some-name-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-2243
++ mktemp
+ local LAST_OUT=/tmp/tmp.NNb7LPPcMt
++ mktemp
+ local LAST_ERR=/tmp/tmp.eol8Qs3kVa
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-2243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-2243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-2243
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.NNb7LPPcMt
+ cat /tmp/tmp.eol8Qs3kVa
error: no matching resources found
+ rm /tmp/tmp.NNb7LPPcMt /tmp/tmp.eol8Qs3kVa
+ return 1
+ true
+ wait_for_running some-name-proxysql 1
+ local name=some-name-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-proxysql-0 480
+ local pod=some-name-proxysql-0
+ local max_retry=480
+ local ns=
++ echo some-name-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
pod/some-name-proxysql-0 condition met
some-name-proxysql-0.Ok
+ wait_for_running some-name-pxc 3
+ local name=some-name-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
some-name-pxc-0.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-1 480
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
some-name-pxc-1.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-2 480
+ local pod=some-name-pxc-2
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-2 condition met
some-name-pxc-2.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-proxysql -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h some-name-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.99wOhowQ7p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Cw9Xe5s0iL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.99wOhowQ7p
++ cat /tmp/tmp.Cw9Xe5s0iL
++ rm /tmp/tmp.99wOhowQ7p /tmp/tmp.Cw9Xe5s0iL
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-proxysql -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h some-name-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wuNvh19l87
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dafGgiHOed
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wuNvh19l87
++ cat /tmp/tmp.dafGgiHOed
++ rm /tmp/tmp.wuNvh19l87 /tmp/tmp.dafGgiHOed
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.btjYqxPk1w
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vj4IypfZFS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.btjYqxPk1w
++ cat /tmp/tmp.vj4IypfZFS
++ rm /tmp/tmp.btjYqxPk1w /tmp/tmp.vj4IypfZFS
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-1.sql /tmp/tmp.0sjIU7TyM0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bmBgE81ZlE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A0ivgoWh8X
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bmBgE81ZlE
++ cat /tmp/tmp.A0ivgoWh8X
++ rm /tmp/tmp.bmBgE81ZlE /tmp/tmp.A0ivgoWh8X
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-1.sql /tmp/tmp.0sjIU7TyM0/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-1-80.sql ']'
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jdj7LSziJc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RUnMPbYrLq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Jdj7LSziJc
++ cat /tmp/tmp.RUnMPbYrLq
++ rm /tmp/tmp.Jdj7LSziJc /tmp/tmp.RUnMPbYrLq
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-1.sql /tmp/tmp.0sjIU7TyM0/select-1.sql
++ is_keyring_plugin_in_use some-name
++ local cluster=some-name
++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W6BtZzhbHc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bQYtJnVl2n
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.W6BtZzhbHc
++ cat /tmp/tmp.bQYtJnVl2n
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.W6BtZzhbHc /tmp/tmp.bQYtJnVl2n
++ return 0
+ '[' '' ']'
+ desc 'test root'
+ set +o xtrace


-----------------------------------------------------------------------------------
test root
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets root dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=root
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.KbofikhZRo
++ mktemp
+ local LAST_ERR=/tmp/tmp.VLFPBmcRm4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.KbofikhZRo
secret/my-cluster-secrets patched
+ cat /tmp/tmp.VLFPBmcRm4
+ rm /tmp/tmp.KbofikhZRo /tmp/tmp.VLFPBmcRm4
+ return 0
+ sleep 15
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DGNglbyT9Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BZg50qN8Kh
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DGNglbyT9Q
++ cat /tmp/tmp.BZg50qN8Kh
++ rm /tmp/tmp.DGNglbyT9Q /tmp/tmp.BZg50qN8Kh
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.0sjIU7TyM0/select-4.sql
+ desc 'test proxyadmin'
+ set +o xtrace


-----------------------------------------------------------------------------------
test proxyadmin
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.xHPUoiaxMd
++ mktemp
+ local LAST_ERR=/tmp/tmp.DgLI55hwnE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xHPUoiaxMd
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.DgLI55hwnE
+ rm /tmp/tmp.xHPUoiaxMd /tmp/tmp.DgLI55hwnE
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PcejjXqJG0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3EvphhN0jR
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PcejjXqJG0
++ cat /tmp/tmp.3EvphhN0jR
++ rm /tmp/tmp.PcejjXqJG0 /tmp/tmp.3EvphhN0jR
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7K7DIXYENA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5waXBeVr3N
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7K7DIXYENA
++ cat /tmp/tmp.5waXBeVr3N
++ rm /tmp/tmp.7K7DIXYENA /tmp/tmp.5waXBeVr3N
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.Ix7rvQxmpX
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.y1Qi4hsjQO
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.Ix7rvQxmpX
+++++ cat /tmp/tmp.y1Qi4hsjQO
+++++ rm /tmp/tmp.Ix7rvQxmpX /tmp/tmp.y1Qi4hsjQO
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.XWAakUWAtY
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.0XmDb7c0AU
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.XWAakUWAtY
+++++ cat /tmp/tmp.0XmDb7c0AU
+++++ rm /tmp/tmp.XWAakUWAtY /tmp/tmp.0XmDb7c0AU
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WFiLcDgYlf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cSeFzYaOHv
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WFiLcDgYlf
++ cat /tmp/tmp.cSeFzYaOHv
++ rm /tmp/tmp.WFiLcDgYlf /tmp/tmp.cSeFzYaOHv
++ return 0
+ [[ 3 == \3 ]]
+ patch_secret my-cluster-secrets proxyadmin dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=proxyadmin
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.7cm73hGGRH
++ mktemp
+ local LAST_ERR=/tmp/tmp.NJYvASmKfI
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7cm73hGGRH
secret/my-cluster-secrets patched
+ cat /tmp/tmp.NJYvASmKfI
+ rm /tmp/tmp.7cm73hGGRH /tmp/tmp.NJYvASmKfI
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IN5V9F7Ys5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YbuY18bKma
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IN5V9F7Ys5
++ cat /tmp/tmp.YbuY18bKma
++ rm /tmp/tmp.IN5V9F7Ys5 /tmp/tmp.YbuY18bKma
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xDAHnIyqm2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8pHuhiKujj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xDAHnIyqm2
++ cat /tmp/tmp.8pHuhiKujj
++ rm /tmp/tmp.xDAHnIyqm2 /tmp/tmp.8pHuhiKujj
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pX28Qbw2Qg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2VfpFtE3zV
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pX28Qbw2Qg
++ cat /tmp/tmp.2VfpFtE3zV
++ rm /tmp/tmp.pX28Qbw2Qg /tmp/tmp.2VfpFtE3zV
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZK8VtyFFA7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TTxl6mpjQj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZK8VtyFFA7
++ cat /tmp/tmp.TTxl6mpjQj
++ rm /tmp/tmp.ZK8VtyFFA7 /tmp/tmp.TTxl6mpjQj
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eeCnWB9kuc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N9kIQvRwk7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eeCnWB9kuc
++ cat /tmp/tmp.N9kIQvRwk7
++ rm /tmp/tmp.eeCnWB9kuc /tmp/tmp.N9kIQvRwk7
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.DkiWbPr5em
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.QjncmQxccG
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.DkiWbPr5em
+++++ cat /tmp/tmp.QjncmQxccG
+++++ rm /tmp/tmp.DkiWbPr5em /tmp/tmp.QjncmQxccG
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.A0j0BnWzcp
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.ezEvmBVgbi
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.A0j0BnWzcp
+++++ cat /tmp/tmp.ezEvmBVgbi
+++++ rm /tmp/tmp.A0j0BnWzcp /tmp/tmp.ezEvmBVgbi
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jr5Ly2No5Z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mfTq5lZebx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Jr5Ly2No5Z
++ cat /tmp/tmp.mfTq5lZebx
++ rm /tmp/tmp.Jr5Ly2No5Z /tmp/tmp.mfTq5lZebx
++ return 0
+ [[ 3 == \3 ]]
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-0
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-2-80.sql ']'
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-0
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-2.sql /tmp/tmp.0sjIU7TyM0/select-2.sql
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-1
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-2-80.sql ']'
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-1
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-2.sql /tmp/tmp.0sjIU7TyM0/select-2.sql
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-2
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-2-80.sql ']'
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-2
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-2.sql /tmp/tmp.0sjIU7TyM0/select-2.sql
+ desc 'test xtrabackup'
+ set +o xtrace


-----------------------------------------------------------------------------------
test xtrabackup
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.QYIimTa74F
++ mktemp
+ local LAST_ERR=/tmp/tmp.VcXspuCmRL
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QYIimTa74F
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.VcXspuCmRL
+ rm /tmp/tmp.QYIimTa74F /tmp/tmp.VcXspuCmRL
+ return 0
+ patch_secret my-cluster-secrets xtrabackup dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=xtrabackup
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.9zxiatETfB
++ mktemp
+ local LAST_ERR=/tmp/tmp.QzgvmEX1Jn
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.9zxiatETfB
secret/my-cluster-secrets patched
+ cat /tmp/tmp.QzgvmEX1Jn
+ rm /tmp/tmp.9zxiatETfB /tmp/tmp.QzgvmEX1Jn
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Yz3vJZwA95
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z59VsGzY3m
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Yz3vJZwA95
++ cat /tmp/tmp.Z59VsGzY3m
++ rm /tmp/tmp.Yz3vJZwA95 /tmp/tmp.Z59VsGzY3m
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wLQ45k0KjX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OEPPYcjJUF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wLQ45k0KjX
++ cat /tmp/tmp.OEPPYcjJUF
++ rm /tmp/tmp.wLQ45k0KjX /tmp/tmp.OEPPYcjJUF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.houTqZrP3f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.E7bsWAHmeQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.houTqZrP3f
++ cat /tmp/tmp.E7bsWAHmeQ
++ rm /tmp/tmp.houTqZrP3f /tmp/tmp.E7bsWAHmeQ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.e6ZiOBJalF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EODZVtLRw2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.e6ZiOBJalF
++ cat /tmp/tmp.EODZVtLRw2
++ rm /tmp/tmp.e6ZiOBJalF /tmp/tmp.EODZVtLRw2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gKQ4gDw4fy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hsHK3qa63o
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gKQ4gDw4fy
++ cat /tmp/tmp.hsHK3qa63o
++ rm /tmp/tmp.gKQ4gDw4fy /tmp/tmp.hsHK3qa63o
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LE151wWvyT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cKwMyoNqQI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LE151wWvyT
++ cat /tmp/tmp.cKwMyoNqQI
++ rm /tmp/tmp.LE151wWvyT /tmp/tmp.cKwMyoNqQI
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uyWW1lG0jG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.na0hcIVZlC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.uyWW1lG0jG
++ cat /tmp/tmp.na0hcIVZlC
++ rm /tmp/tmp.uyWW1lG0jG /tmp/tmp.na0hcIVZlC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 6 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BM61YziOHv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KMq6RLmzPp
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BM61YziOHv
++ cat /tmp/tmp.KMq6RLmzPp
++ rm /tmp/tmp.BM61YziOHv /tmp/tmp.KMq6RLmzPp
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hXme4b48gI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H81xDXJaHb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hXme4b48gI
++ cat /tmp/tmp.H81xDXJaHb
++ rm /tmp/tmp.hXme4b48gI /tmp/tmp.H81xDXJaHb
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.sdIRWza3Le
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.pyyqDSg3lu
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.sdIRWza3Le
+++++ cat /tmp/tmp.pyyqDSg3lu
+++++ rm /tmp/tmp.sdIRWza3Le /tmp/tmp.pyyqDSg3lu
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.q12OzWgqQH
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.ifzRFelL2y
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.q12OzWgqQH
+++++ cat /tmp/tmp.ifzRFelL2y
+++++ rm /tmp/tmp.q12OzWgqQH /tmp/tmp.ifzRFelL2y
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XAduIbl0CX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GjT2H41i2y
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XAduIbl0CX
++ cat /tmp/tmp.GjT2H41i2y
++ rm /tmp/tmp.XAduIbl0CX /tmp/tmp.GjT2H41i2y
++ return 0
+ [[ 2 == \2 ]]
+ compare_mysql_cmd_local select-3 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 '' pxc
+ local command_id=select-3
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\'''
+ local pod=some-name-pxc-0
+ local postfix=
+ local container_name=pxc
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-3-80.sql ']'
+ run_mysql_local 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 pxc
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\'''
+ local pod=some-name-pxc-0
+ local container_name=pxc
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-3.sql /tmp/tmp.0sjIU7TyM0/select-3.sql
+ desc 'test monitor'
+ set +o xtrace


-----------------------------------------------------------------------------------
test monitor
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=monitor
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.5UADPib89P
++ mktemp
+ local LAST_ERR=/tmp/tmp.OrwHdShFPs
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5UADPib89P
secret/my-cluster-secrets patched
+ cat /tmp/tmp.OrwHdShFPs
+ rm /tmp/tmp.5UADPib89P /tmp/tmp.OrwHdShFPs
+ return 0
+ wait_for_password_propagation my-cluster-secrets monitor
+ local secret=my-cluster-secrets
+ local user=monitor
+ local max_retry=240
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.meTM1uqqXo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9VFafT5HqR
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.meTM1uqqXo
++ cat /tmp/tmp.9VFafT5HqR
++ rm /tmp/tmp.meTM1uqqXo /tmp/tmp.9VFafT5HqR
++ return 0
+ local root_pass=test-password
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ retry=0
+ is_password_updated monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ grep additional_password
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fclemdJKGA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IbTjbwGNa8
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.fclemdJKGA
++ cat /tmp/tmp.IbTjbwGNa8
++ rm /tmp/tmp.fclemdJKGA /tmp/tmp.IbTjbwGNa8
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ echo 'waiting for password update'
waiting for password update
+ sleep 1
+ let retry+=1
+ [[ 1 -ge 240 ]]
+ is_password_updated monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep additional_password
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pVHzDaBFO8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o94R8S57dc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pVHzDaBFO8
++ cat /tmp/tmp.o94R8S57dc
++ rm /tmp/tmp.pVHzDaBFO8 /tmp/tmp.o94R8S57dc
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
{"additional_password": "*C1F414D9BAF378B656A849B31F9F8AF3125F558B"}
+ retry=0
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X2BNISKWKa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.01GX4LeJAo
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.X2BNISKWKa
++ cat /tmp/tmp.01GX4LeJAo
++ rm /tmp/tmp.X2BNISKWKa /tmp/tmp.01GX4LeJAo
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 1 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XWATjSCDiR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HSTYMui2fS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XWATjSCDiR
++ cat /tmp/tmp.HSTYMui2fS
++ rm /tmp/tmp.XWATjSCDiR /tmp/tmp.HSTYMui2fS
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 2 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KpX89AV3Ay
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JD5y43pj5B
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KpX89AV3Ay
++ cat /tmp/tmp.JD5y43pj5B
++ rm /tmp/tmp.KpX89AV3Ay /tmp/tmp.JD5y43pj5B
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 3 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XjGM5IGrOb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RZZcq2Df7B
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XjGM5IGrOb
++ cat /tmp/tmp.RZZcq2Df7B
++ rm /tmp/tmp.XjGM5IGrOb /tmp/tmp.RZZcq2Df7B
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 4 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eQrBKpZ8tk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IezhkoSR6Q
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eQrBKpZ8tk
++ cat /tmp/tmp.IezhkoSR6Q
++ rm /tmp/tmp.eQrBKpZ8tk /tmp/tmp.IezhkoSR6Q
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 5 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ grep NULL
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PMZ1aXRxqa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TcwVVe983v
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PMZ1aXRxqa
++ cat /tmp/tmp.TcwVVe983v
++ rm /tmp/tmp.PMZ1aXRxqa /tmp/tmp.TcwVVe983v
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 6 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PV8KLyguGR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BPQ7MzzH31
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PV8KLyguGR
++ cat /tmp/tmp.BPQ7MzzH31
++ rm /tmp/tmp.PV8KLyguGR /tmp/tmp.BPQ7MzzH31
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 7 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BaWq7BmRmG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ktM2HNQJkT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BaWq7BmRmG
++ cat /tmp/tmp.ktM2HNQJkT
++ rm /tmp/tmp.BaWq7BmRmG /tmp/tmp.ktM2HNQJkT
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ echo 'waiting for password propagation'
waiting for password propagation
+ sleep 1
+ let retry+=1
+ [[ 8 -ge 240 ]]
+ is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local username=monitor
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\'''
+ local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\'''
+ local 'uri=-h some-name-pxc -uroot -p'\''test-password'\'''
+ grep NULL
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5dRNu3wPz6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NwQ7kvgBlo
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.5dRNu3wPz6
++ cat /tmp/tmp.NwQ7kvgBlo
++ rm /tmp/tmp.5dRNu3wPz6 /tmp/tmp.NwQ7kvgBlo
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
NULL
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3LlvCrt1yt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TF7a1sGDRX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3LlvCrt1yt
++ cat /tmp/tmp.TF7a1sGDRX
++ rm /tmp/tmp.3LlvCrt1yt /tmp/tmp.TF7a1sGDRX
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H6N9TXCxjl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LJGvq970is
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.H6N9TXCxjl
++ cat /tmp/tmp.LJGvq970is
++ rm /tmp/tmp.H6N9TXCxjl /tmp/tmp.LJGvq970is
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.huj2tbYmn2
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.qBQWK8O2Pc
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.huj2tbYmn2
+++++ cat /tmp/tmp.qBQWK8O2Pc
+++++ rm /tmp/tmp.huj2tbYmn2 /tmp/tmp.qBQWK8O2Pc
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.CKHVFLh6zv
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.oWIBIspc4c
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.CKHVFLh6zv
+++++ cat /tmp/tmp.oWIBIspc4c
+++++ rm /tmp/tmp.CKHVFLh6zv /tmp/tmp.oWIBIspc4c
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WVSj8MC0UU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zvLONWLlQQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WVSj8MC0UU
++ cat /tmp/tmp.zvLONWLlQQ
++ rm /tmp/tmp.WVSj8MC0UU /tmp/tmp.zvLONWLlQQ
++ return 0
+ [[ 2 == \2 ]]
+ sleep 10
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TPK5CNsZ9J
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J1vKVrQfqG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TPK5CNsZ9J
++ cat /tmp/tmp.J1vKVrQfqG
++ rm /tmp/tmp.TPK5CNsZ9J /tmp/tmp.J1vKVrQfqG
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.0sjIU7TyM0/select-4.sql
+ desc 'test operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
test operator
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets operator dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=operator
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.fufJAEdiUd
++ mktemp
+ local LAST_ERR=/tmp/tmp.6Q1tJlsRYA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.fufJAEdiUd
secret/my-cluster-secrets patched
+ cat /tmp/tmp.6Q1tJlsRYA
+ rm /tmp/tmp.fufJAEdiUd /tmp/tmp.6Q1tJlsRYA
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CwrBrIi4ZX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PZbmHJDdZd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.CwrBrIi4ZX
++ cat /tmp/tmp.PZbmHJDdZd
++ rm /tmp/tmp.CwrBrIi4ZX /tmp/tmp.PZbmHJDdZd
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DyE9gGwGjr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SJBqWpav8s
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DyE9gGwGjr
++ cat /tmp/tmp.SJBqWpav8s
++ rm /tmp/tmp.DyE9gGwGjr /tmp/tmp.SJBqWpav8s
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eivCObNrdu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ixnc00AVTW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eivCObNrdu
++ cat /tmp/tmp.ixnc00AVTW
++ rm /tmp/tmp.eivCObNrdu /tmp/tmp.ixnc00AVTW
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.tLTaujuaII
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.xYJPi2pi9m
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.tLTaujuaII
+++++ cat /tmp/tmp.xYJPi2pi9m
+++++ rm /tmp/tmp.tLTaujuaII /tmp/tmp.xYJPi2pi9m
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.bgj4AJeNkn
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.rVrpvRmqrO
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.bgj4AJeNkn
+++++ cat /tmp/tmp.rVrpvRmqrO
+++++ rm /tmp/tmp.bgj4AJeNkn /tmp/tmp.rVrpvRmqrO
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WBAkEIsg5r
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wDBD0qAv2g
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WBAkEIsg5r
++ cat /tmp/tmp.wDBD0qAv2g
++ rm /tmp/tmp.WBAkEIsg5r /tmp/tmp.wDBD0qAv2g
++ return 0
+ [[ 2 == \2 ]]
+ sleep 10
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FyODhcBELD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cLseTfB1a9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FyODhcBELD
++ cat /tmp/tmp.cLseTfB1a9
++ rm /tmp/tmp.FyODhcBELD /tmp/tmp.cLseTfB1a9
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.0sjIU7TyM0/select-4.sql
+ desc 'change secret name'
+ set +o xtrace


-----------------------------------------------------------------------------------
change secret name
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.lInVB3YTKG
++ mktemp
+ local LAST_ERR=/tmp/tmp.DZu0SlXvgq
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lInVB3YTKG
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.DZu0SlXvgq
+ rm /tmp/tmp.lInVB3YTKG /tmp/tmp.DZu0SlXvgq
+ return 0
+ sleep 30
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w8yhPwT85l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sGHwsIYsox
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.w8yhPwT85l
++ cat /tmp/tmp.sGHwsIYsox
++ rm /tmp/tmp.w8yhPwT85l /tmp/tmp.sGHwsIYsox
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nahdi7jzmb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kDcXiQa97Z
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nahdi7jzmb
++ cat /tmp/tmp.kDcXiQa97Z
++ rm /tmp/tmp.nahdi7jzmb /tmp/tmp.kDcXiQa97Z
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ccY6tuVsPI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YFIh8zaSb2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ccY6tuVsPI
++ cat /tmp/tmp.YFIh8zaSb2
++ rm /tmp/tmp.ccY6tuVsPI /tmp/tmp.YFIh8zaSb2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.49MmJYP0Te
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PWllkCHi1i
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.49MmJYP0Te
++ cat /tmp/tmp.PWllkCHi1i
++ rm /tmp/tmp.49MmJYP0Te /tmp/tmp.PWllkCHi1i
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CCOF1Qn8ly
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b52p2vydiG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.CCOF1Qn8ly
++ cat /tmp/tmp.b52p2vydiG
++ rm /tmp/tmp.CCOF1Qn8ly /tmp/tmp.b52p2vydiG
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bmXUVrxI4f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vBtpBbhjH8
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bmXUVrxI4f
++ cat /tmp/tmp.vBtpBbhjH8
++ rm /tmp/tmp.bmXUVrxI4f /tmp/tmp.vBtpBbhjH8
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.csGXLKSyhk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XKAmNXgVF8
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.csGXLKSyhk
++ cat /tmp/tmp.XKAmNXgVF8
++ rm /tmp/tmp.csGXLKSyhk /tmp/tmp.XKAmNXgVF8
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X6aUF9qSfQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.otKREGWz5J
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.X6aUF9qSfQ
++ cat /tmp/tmp.otKREGWz5J
++ rm /tmp/tmp.X6aUF9qSfQ /tmp/tmp.otKREGWz5J
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.TIEOpTrxkw
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.hOKOQVv5gK
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.TIEOpTrxkw
+++++ cat /tmp/tmp.hOKOQVv5gK
+++++ rm /tmp/tmp.TIEOpTrxkw /tmp/tmp.hOKOQVv5gK
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.uCnnHJV7qQ
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.zRVyEVXE7E
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.uCnnHJV7qQ
+++++ cat /tmp/tmp.zRVyEVXE7E
+++++ rm /tmp/tmp.uCnnHJV7qQ /tmp/tmp.zRVyEVXE7E
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mdtqUUON2S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b3LDzlOlXo
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.mdtqUUON2S
++ cat /tmp/tmp.b3LDzlOlXo
++ rm /tmp/tmp.mdtqUUON2S /tmp/tmp.b3LDzlOlXo
++ return 0
+ [[ 2 == \2 ]]
+ desc 'test new operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
test new operator
-----------------------------------------------------------------------------------

+ newpass=test-password2
++ echo -n test-password2
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZDI=
+ patch_secret my-cluster-secrets-2 operator dGVzdC1wYXNzd29yZDI=
+ local secret=my-cluster-secrets-2
+ local key=operator
+ local value=dGVzdC1wYXNzd29yZDI=
+ kubectl_bin patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.O8TSS2HOpM
++ mktemp
+ local LAST_ERR=/tmp/tmp.Jk37myvNo0
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.O8TSS2HOpM
secret/my-cluster-secrets-2 patched
+ cat /tmp/tmp.Jk37myvNo0
+ rm /tmp/tmp.O8TSS2HOpM /tmp/tmp.Jk37myvNo0
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kfewdSLk5Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ftdF0IoGV3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kfewdSLk5Q
++ cat /tmp/tmp.ftdF0IoGV3
++ rm /tmp/tmp.kfewdSLk5Q /tmp/tmp.ftdF0IoGV3
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dOoyln6f3V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qbCB3ZRq1p
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.dOoyln6f3V
++ cat /tmp/tmp.qbCB3ZRq1p
++ rm /tmp/tmp.dOoyln6f3V /tmp/tmp.qbCB3ZRq1p
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PMyIPP6z4l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.quBQTD2Q6R
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PMyIPP6z4l
++ cat /tmp/tmp.quBQTD2Q6R
++ rm /tmp/tmp.PMyIPP6z4l /tmp/tmp.quBQTD2Q6R
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.ZW6i4CeSJI
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.gHMlgI5HNX
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.ZW6i4CeSJI
+++++ cat /tmp/tmp.gHMlgI5HNX
+++++ rm /tmp/tmp.ZW6i4CeSJI /tmp/tmp.gHMlgI5HNX
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.05vUv5yU4S
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.5J5oo72mlA
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.05vUv5yU4S
+++++ cat /tmp/tmp.5J5oo72mlA
+++++ rm /tmp/tmp.05vUv5yU4S /tmp/tmp.5J5oo72mlA
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ToOBMcle2m
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DL6p8VEuZ3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ToOBMcle2m
++ cat /tmp/tmp.DL6p8VEuZ3
++ rm /tmp/tmp.ToOBMcle2m /tmp/tmp.DL6p8VEuZ3
++ return 0
+ [[ 2 == \2 ]]
+ sleep 20
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GKr4pM06uW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rRTMFdFqJF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GKr4pM06uW
++ cat /tmp/tmp.rRTMFdFqJF
++ rm /tmp/tmp.GKr4pM06uW /tmp/tmp.rRTMFdFqJF
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.0sjIU7TyM0/select-4.sql
++ getSecretData my-cluster-secrets-2 root
++ local secretName=my-cluster-secrets-2
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets-2 '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WlhVTxPuQZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NJVEsled9X
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets-2 '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WlhVTxPuQZ
++ cat /tmp/tmp.NJVEsled9X
++ rm /tmp/tmp.WlhVTxPuQZ /tmp/tmp.NJVEsled9X
++ return 0
+ newpass=9ZiM=PjoQWdC55hNfk
+ desc 'test new users sync'
+ set +o xtrace


-----------------------------------------------------------------------------------
test new users sync
-----------------------------------------------------------------------------------

+ run_mysql 'CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\''9ZiM=PjoQWdC55hNfk'\'';' '-h some-name-pxc -uroot -p'\''9ZiM=PjoQWdC55hNfk'\'''
+ local 'command=CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\''9ZiM=PjoQWdC55hNfk'\'';'
+ local 'uri=-h some-name-pxc -uroot -p'\''9ZiM=PjoQWdC55hNfk'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ai1ECCuzkg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Rk2I2wgHyn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Ai1ECCuzkg
++ cat /tmp/tmp.Rk2I2wgHyn
++ rm /tmp/tmp.Ai1ECCuzkg /tmp/tmp.Rk2I2wgHyn
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ sleep 40
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\''9ZiM=PjoQWdC55hNfk'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -utestsync -p'\''9ZiM=PjoQWdC55hNfk'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\''9ZiM=PjoQWdC55hNfk'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -utestsync -p'\''9ZiM=PjoQWdC55hNfk'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lsbNPd1OUq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TZJdwRKy0E
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.lsbNPd1OUq
++ cat /tmp/tmp.TZJdwRKy0E
++ rm /tmp/tmp.lsbNPd1OUq /tmp/tmp.TZJdwRKy0E
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.0sjIU7TyM0/select-4.sql
++ getSecretData internal-some-name operator
++ local secretName=internal-some-name
++ local dataKey=operator
++ kubectl_bin get secrets/internal-some-name '--template={{.data.operator}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CbrcGp1mSC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fU8xUBxmde
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/internal-some-name '--template={{.data.operator}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.CbrcGp1mSC
++ cat /tmp/tmp.fU8xUBxmde
++ rm /tmp/tmp.CbrcGp1mSC /tmp/tmp.fU8xUBxmde
++ return 0
+ pass=test-password2
+ desc 'check secret without operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
check secret without operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ltt1TDITSt
++ mktemp
+ local LAST_ERR=/tmp/tmp.pDq1U50hR6
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Ltt1TDITSt
secret/my-cluster-secrets-2 configured
+ cat /tmp/tmp.pDq1U50hR6
Warning: resource secrets/my-cluster-secrets-2 is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.Ltt1TDITSt /tmp/tmp.pDq1U50hR6
+ return 0
+ sleep 15
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql ']'
+ expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P4FuJflBAQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dSG8kF1dSO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.P4FuJflBAQ
++ cat /tmp/tmp.dSG8kF1dSO
++ rm /tmp/tmp.P4FuJflBAQ /tmp/tmp.dSG8kF1dSO
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.0sjIU7TyM0/select-4.sql
+ newpass=test-password2
++ echo -n test-password2
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZDI=
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/conf/some-name.yml
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ local LAST_OUT=/tmp/tmp.4qnGqNohc3
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1717-958d148d#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-2243~
+ local LAST_ERR=/tmp/tmp.LDCASSjDeU
+ local exit_status=0
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.4qnGqNohc3
perconaxtradbcluster.pxc.percona.com/some-name configured
+ cat /tmp/tmp.LDCASSjDeU
+ rm /tmp/tmp.4qnGqNohc3 /tmp/tmp.LDCASSjDeU
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sG5FyMPYJH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VBGhLeqWB2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sG5FyMPYJH
++ cat /tmp/tmp.VBGhLeqWB2
++ rm /tmp/tmp.sG5FyMPYJH /tmp/tmp.VBGhLeqWB2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LsdLDyIO6l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WUl7AnLfgm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LsdLDyIO6l
++ cat /tmp/tmp.WUl7AnLfgm
++ rm /tmp/tmp.LsdLDyIO6l /tmp/tmp.WUl7AnLfgm
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l1VzSblnxH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.76zNjCK3pQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.l1VzSblnxH
++ cat /tmp/tmp.76zNjCK3pQ
++ rm /tmp/tmp.l1VzSblnxH /tmp/tmp.76zNjCK3pQ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BqQffjSJkK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JztsZdFuvu
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BqQffjSJkK
++ cat /tmp/tmp.JztsZdFuvu
++ rm /tmp/tmp.BqQffjSJkK /tmp/tmp.JztsZdFuvu
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ECh5IUuE2T
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bS6YaNd3xB
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ECh5IUuE2T
++ cat /tmp/tmp.bS6YaNd3xB
++ rm /tmp/tmp.ECh5IUuE2T /tmp/tmp.bS6YaNd3xB
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.woweAuym6P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jD1bZd3Rps
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.woweAuym6P
++ cat /tmp/tmp.jD1bZd3Rps
++ rm /tmp/tmp.woweAuym6P /tmp/tmp.jD1bZd3Rps
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S8cxHZxZnC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MfyRKS4Zbo
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.S8cxHZxZnC
++ cat /tmp/tmp.MfyRKS4Zbo
++ rm /tmp/tmp.S8cxHZxZnC /tmp/tmp.MfyRKS4Zbo
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 6 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uKlV0sWqta
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fJWgMVQ1bz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.uKlV0sWqta
++ cat /tmp/tmp.fJWgMVQ1bz
++ rm /tmp/tmp.uKlV0sWqta /tmp/tmp.fJWgMVQ1bz
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 7 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.51odztDzgJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7Pu75OQHqK
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.51odztDzgJ
++ cat /tmp/tmp.7Pu75OQHqK
++ rm /tmp/tmp.51odztDzgJ /tmp/tmp.7Pu75OQHqK
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 8 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Cs5ba43flu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uQDOcUhVZ9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Cs5ba43flu
++ cat /tmp/tmp.uQDOcUhVZ9
++ rm /tmp/tmp.Cs5ba43flu /tmp/tmp.uQDOcUhVZ9
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 9 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UGjmCKJMUU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H9XSI9UAEx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.UGjmCKJMUU
++ cat /tmp/tmp.H9XSI9UAEx
++ rm /tmp/tmp.UGjmCKJMUU /tmp/tmp.H9XSI9UAEx
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 10 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bMbr6a65Rf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nYY15YThHG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bMbr6a65Rf
++ cat /tmp/tmp.nYY15YThHG
++ rm /tmp/tmp.bMbr6a65Rf /tmp/tmp.nYY15YThHG
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 11 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yOtyMDRqKk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ravzm4fj9z
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.yOtyMDRqKk
++ cat /tmp/tmp.ravzm4fj9z
++ rm /tmp/tmp.yOtyMDRqKk /tmp/tmp.ravzm4fj9z
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 12 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.h1RBRmmwZk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cbetQZ1B8m
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.h1RBRmmwZk
++ cat /tmp/tmp.cbetQZ1B8m
++ rm /tmp/tmp.h1RBRmmwZk /tmp/tmp.cbetQZ1B8m
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 13 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JS0TUDzlIo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DlkzPRXuZW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.JS0TUDzlIo
++ cat /tmp/tmp.DlkzPRXuZW
++ rm /tmp/tmp.JS0TUDzlIo /tmp/tmp.DlkzPRXuZW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 14 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MdgbdLpJJT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GLXxteSttH
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.MdgbdLpJJT
++ cat /tmp/tmp.GLXxteSttH
++ rm /tmp/tmp.MdgbdLpJJT /tmp/tmp.GLXxteSttH
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BfivKO88vq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rA12eRMAsN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BfivKO88vq
++ cat /tmp/tmp.rA12eRMAsN
++ rm /tmp/tmp.BfivKO88vq /tmp/tmp.rA12eRMAsN
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.oHYuOsxdRl
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.Mkd6Me1lgC
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.oHYuOsxdRl
+++++ cat /tmp/tmp.Mkd6Me1lgC
+++++ rm /tmp/tmp.oHYuOsxdRl /tmp/tmp.Mkd6Me1lgC
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.r8Ml3IPnjG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QYRtY5DNue
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.r8Ml3IPnjG
++ cat /tmp/tmp.QYRtY5DNue
++ rm /tmp/tmp.r8Ml3IPnjG /tmp/tmp.QYRtY5DNue
++ return 0
+ [[ 3 == \3 ]]
+ patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZDI=
+ local secret=my-cluster-secrets
+ local key=monitor
+ local value=dGVzdC1wYXNzd29yZDI=
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.G3O8QSV8Y3
++ mktemp
+ local LAST_ERR=/tmp/tmp.mCnmtaT4IO
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.G3O8QSV8Y3
secret/my-cluster-secrets patched
+ cat /tmp/tmp.mCnmtaT4IO
+ rm /tmp/tmp.G3O8QSV8Y3 /tmp/tmp.mCnmtaT4IO
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.INET9L1A32
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2czWbCu4N4
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.INET9L1A32
++ cat /tmp/tmp.2czWbCu4N4
++ rm /tmp/tmp.INET9L1A32 /tmp/tmp.2czWbCu4N4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BCOmSFOFAC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3y5GpujbqS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BCOmSFOFAC
++ cat /tmp/tmp.3y5GpujbqS
++ rm /tmp/tmp.BCOmSFOFAC /tmp/tmp.3y5GpujbqS
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7auq0Ze69n
+++ mktemp
++ local LAST_ERR=/tmp/tmp.URpcTjgDHl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7auq0Ze69n
++ cat /tmp/tmp.URpcTjgDHl
++ rm /tmp/tmp.7auq0Ze69n /tmp/tmp.URpcTjgDHl
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t8kzQE56Pj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EG671Fk3yL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.t8kzQE56Pj
++ cat /tmp/tmp.EG671Fk3yL
++ rm /tmp/tmp.t8kzQE56Pj /tmp/tmp.EG671Fk3yL
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g8GanNvtLh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kyjnHQ6cAE
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.g8GanNvtLh
++ cat /tmp/tmp.kyjnHQ6cAE
++ rm /tmp/tmp.g8GanNvtLh /tmp/tmp.kyjnHQ6cAE
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.KwDQYz8r4s
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.JMSEZQQnyw
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.KwDQYz8r4s
+++++ cat /tmp/tmp.JMSEZQQnyw
+++++ rm /tmp/tmp.KwDQYz8r4s /tmp/tmp.JMSEZQQnyw
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pDAdl5eFtd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2ISckabgLe
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pDAdl5eFtd
++ cat /tmp/tmp.2ISckabgLe
++ rm /tmp/tmp.pDAdl5eFtd /tmp/tmp.2ISckabgLe
++ return 0
+ [[ 3 == \3 ]]
+ compare_mysql_cmd select-3 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local command_id=select-3
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-3-80.sql ']'
+ run_mysql 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ixZXcJu5jx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.op1f1iCDOh
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ixZXcJu5jx
++ cat /tmp/tmp.op1f1iCDOh
++ rm /tmp/tmp.ixZXcJu5jx /tmp/tmp.op1f1iCDOh
++ return 0
+ client_pod=pxc-client-6644d8898f-9sngm
+ wait_pod pxc-client-6644d8898f-9sngm
+ local pod=pxc-client-6644d8898f-9sngm
+ local max_retry=480
+ local ns=
++ echo pxc-client-6644d8898f-9sngm
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6644d8898f-9sngm condition met
pxc-client-6644d8898f-9sngm.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.0sjIU7TyM0/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1717/e2e-tests/users/compare/select-3.sql /tmp/tmp.0sjIU7TyM0/select-3.sql
+ destroy users-2243
+ local namespace=users-2243
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ grep -v 'the object has been modified'
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ grep -v 'get backup status: Job.batch'
+ grep -v level=info
+ sort -u
+ tee /tmp/tmp.0sjIU7TyM0/operator.log
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BdiIpKHpbL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CcS9bCNgTa
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BdiIpKHpbL
++ cat /tmp/tmp.CcS9bCNgTa
++ rm /tmp/tmp.BdiIpKHpbL /tmp/tmp.CcS9bCNgTa
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-76cddf6977-dz79t
++ mktemp
+ local LAST_OUT=/tmp/tmp.6byJbUXOPU
++ mktemp
+ local LAST_ERR=/tmp/tmp.KEUXZMA0KZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-76cddf6977-dz79t
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6byJbUXOPU
+ cat /tmp/tmp.KEUXZMA0KZ
+ rm /tmp/tmp.6byJbUXOPU /tmp/tmp.KEUXZMA0KZ
+ return 0
2024-05-30T03:49:22.751Z	INFO	setup	Manager starting up	{"gitCommit": "958d148d923a36c35b9549e7bd3710ce6dadf796", "gitBranch": "PR-1717-958d148d", "buildTime": "2024-05-30T01:47:55Z", "goVersion": "go1.22.3", "os": "linux", "arch": "amd64"}
2024-05-30T03:49:22.751Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.26.15-gke.1360000"}
2024-05-30T03:49:22.752Z	INFO	setup	Registering Components.
2024-05-30T03:49:27.024Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2024-05-30T03:49:27.027Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2024-05-30T03:49:27.027Z	INFO	controller-runtime.metrics	Starting metrics server
2024-05-30T03:49:27.027Z	INFO	controller-runtime.webhook	Starting webhook server
2024-05-30T03:49:27.027Z	INFO	setup	Starting the Cmd.
2024-05-30T03:49:27.027Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2024-05-30T03:49:27.028Z	INFO	controller-runtime.certwatcher	Starting certificate watcher
2024-05-30T03:49:27.028Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate
2024-05-30T03:49:27.028Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2024-05-30T03:49:27.128Z	INFO	attempting to acquire leader lease pxc-operator/08db1feb.percona.com...
2024-05-30T03:49:27.148Z	DEBUG	events	percona-xtradb-cluster-operator-76cddf6977-dz79t_873b4681-0de5-48df-842b-6f0f73805166 became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"b0933a6e-8d6d-4cbe-94bb-29c52ee9fbe5","apiVersion":"coordination.k8s.io/v1","resourceVersion":"66271"}, "reason": "LeaderElection"}
2024-05-30T03:49:27.148Z	INFO	successfully acquired lease pxc-operator/08db1feb.percona.com
2024-05-30T03:49:27.149Z	INFO	Starting Controller	{"controller": "pxcbackup-controller"}
2024-05-30T03:49:27.149Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2024-05-30T03:49:27.149Z	INFO	Starting Controller	{"controller": "pxcrestore-controller"}
2024-05-30T03:49:27.149Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2024-05-30T03:49:27.149Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2024-05-30T03:49:27.149Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2024-05-30T03:49:27.255Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "worker count": 1}
2024-05-30T03:49:27.255Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2024-05-30T03:49:27.255Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "worker count": 1}
2024-05-30T03:49:55.038Z	INFO	Set CR version	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "8fbf1aac-e495-4f51-a15e-d685773b4190", "version": "1.15.0"}
2024-05-30T03:51:15.009Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "71c8adcc-8c00-4ffe-b5c2-ff5d21514f2c", "user": "operator"}
2024-05-30T03:51:15.048Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "71c8adcc-8c00-4ffe-b5c2-ff5d21514f2c", "user": "monitor"}
2024-05-30T03:51:15.153Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "71c8adcc-8c00-4ffe-b5c2-ff5d21514f2c"}
2024-05-30T03:51:15.194Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "71c8adcc-8c00-4ffe-b5c2-ff5d21514f2c"}
2024-05-30T03:51:15.243Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "71c8adcc-8c00-4ffe-b5c2-ff5d21514f2c", "user": "xtrabackup"}
2024-05-30T03:51:15.306Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "71c8adcc-8c00-4ffe-b5c2-ff5d21514f2c"}
2024-05-30T03:51:15.344Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "71c8adcc-8c00-4ffe-b5c2-ff5d21514f2c", "user": "replication"}
2024-05-30T03:51:15.440Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "71c8adcc-8c00-4ffe-b5c2-ff5d21514f2c", "err": "get primary pxc pod: not found"}
2024-05-30T03:51:20.087Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "31557c54-df64-428c-8bf3-31697c9a44b0", "err": "get primary pxc pod: not found"}
2024-05-30T03:51:25.293Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "96ec5377-2517-4481-b11d-60a8ef0e07f7", "err": "get primary pxc pod: not found"}
2024-05-30T03:51:30.538Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "cbc08a63-e01b-4dbe-87ec-ff7fc7576614", "err": "get primary pxc pod: not found"}
2024-05-30T03:53:44.164Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "23762c5b-8175-44c1-b895-0770a9ae5ec1", "user": "root"}
2024-05-30T03:53:44.464Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "23762c5b-8175-44c1-b895-0770a9ae5ec1", "new version": "8.0.36-28.1"}
2024-05-30T03:53:47.696Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "23762c5b-8175-44c1-b895-0770a9ae5ec1"}
2024-05-30T03:53:52.666Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "05d5a01d-d3b9-4ce0-b62c-5691feba5588"}
2024-05-30T03:53:58.089Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "2d77b20c-2c3b-449f-8e58-2e2cc5d800f4"}
2024-05-30T03:54:03.657Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "aefa9240-5ffa-42ff-99d3-b29eaea3e021"}
2024-05-30T03:54:08.781Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c48a79bb-bf2f-4f76-ad7a-981b04732a90"}
2024-05-30T03:54:14.694Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "a8e41657-db5f-457e-a3a8-0ac8f447cc0b"}
2024-05-30T03:54:19.765Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "04ccfefd-1432-46cf-8989-b0efdebaac95"}
2024-05-30T03:54:25.258Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "ad002b98-ba75-4b64-9162-d8e7b8dc084b"}
2024-05-30T03:54:30.576Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "e7ba8dcd-bcdc-47ff-8a16-2152ec3b1079"}
2024-05-30T03:54:35.780Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c5d6bde3-7166-4822-b540-11a77c58907f"}
2024-05-30T03:54:41.581Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "b149154f-1503-49e8-9721-948c7764a9c3"}
2024-05-30T03:54:46.968Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "987e5f51-e088-44bc-a766-ad52653ab8f3"}
2024-05-30T03:54:52.273Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "81c41133-1fe4-41b5-ab48-ef345a74a523"}
2024-05-30T03:55:05.004Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "47695af7-0fd6-4a6a-b9f2-959040f0346a", "user": "root"}
2024-05-30T03:55:05.056Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "47695af7-0fd6-4a6a-b9f2-959040f0346a", "user": "root"}
2024-05-30T03:55:05.065Z	INFO	MySQL init secret created	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "47695af7-0fd6-4a6a-b9f2-959040f0346a", "secret": "some-name-mysql-init", "user": "root"}
2024-05-30T03:55:10.415Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "47695af7-0fd6-4a6a-b9f2-959040f0346a"}
2024-05-30T03:55:10.428Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "47695af7-0fd6-4a6a-b9f2-959040f0346a", "user": "root"}
2024-05-30T03:55:10.487Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "47695af7-0fd6-4a6a-b9f2-959040f0346a", "user": "root"}
2024-05-30T03:55:15.007Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "47695af7-0fd6-4a6a-b9f2-959040f0346a"}
2024-05-30T03:55:32.110Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "33f87507-1873-4813-9545-759ce2cd72bc", "err": "get primary pxc pod: not found"}
2024-05-30T03:55:32.468Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9cdf641b-5b03-40e5-9b6f-a620ec04f1b8", "err": "get primary pxc pod: not found"}
2024-05-30T03:55:36.580Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9cdf641b-5b03-40e5-9b6f-a620ec04f1b8", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-05-30T03:55:41.627Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "ecf5ef09-bc9d-49ab-b992-eb386771076c", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-05-30T03:55:43.162Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "410677ab-b576-4e7b-8f9f-f30745a51809", "user": "proxyadmin"}
2024-05-30T03:55:43.162Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "410677ab-b576-4e7b-8f9f-f30745a51809", "user": "proxyadmin"}
2024-05-30T03:55:43.237Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "410677ab-b576-4e7b-8f9f-f30745a51809", "user": "proxyadmin"}
2024-05-30T03:55:43.251Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "410677ab-b576-4e7b-8f9f-f30745a51809", "user": "proxyadmin"}
2024-05-30T03:55:43.251Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "410677ab-b576-4e7b-8f9f-f30745a51809", "last-applied-secret": "1a55867d0f4fa0d0643653b2eb57b2f690315d8d843a10599c9839c2a779a95f"}
2024-05-30T03:55:43.481Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "410677ab-b576-4e7b-8f9f-f30745a51809", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-05-30T03:55:54.485Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "d2f4bafc-fe22-478b-910a-783e75a3a11a", "err": "get primary pxc pod: not found"}
2024-05-30T03:55:59.702Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "2864dea0-8e66-449b-b3f1-965d400e7ec5", "err": "get primary pxc pod: not found"}
2024-05-30T03:56:04.912Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "b33abc4f-9ad2-4bbc-a700-4045884c9b7b", "err": "get primary pxc pod: not found"}
2024-05-30T03:56:52.337Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "b8375cdd-b991-4a2e-b9a7-d9b0f46c10a2"}
2024-05-30T03:57:02.751Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "ff4eeb5d-070a-47a8-9156-ff3988603a0b"}
2024-05-30T03:57:08.869Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "b212aabe-24a1-4ab9-8538-429b2e74fdb3"}
2024-05-30T03:57:12.746Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c6f5f70f-fa0f-4180-b49d-c8b43bb610d2", "user": "xtrabackup"}
2024-05-30T03:57:12.783Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c6f5f70f-fa0f-4180-b49d-c8b43bb610d2", "user": "xtrabackup"}
2024-05-30T03:57:12.851Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c6f5f70f-fa0f-4180-b49d-c8b43bb610d2", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-05-30T03:57:12.910Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c6f5f70f-fa0f-4180-b49d-c8b43bb610d2", "user": "xtrabackup"}
2024-05-30T03:57:12.940Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c6f5f70f-fa0f-4180-b49d-c8b43bb610d2", "user": "xtrabackup"}
2024-05-30T03:57:12.951Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c6f5f70f-fa0f-4180-b49d-c8b43bb610d2", "last-applied-secret": "d11bcc70ca53ea410cd455e13d82e7225917b0861d9b677ed34c02c5f6fbaf1b"}
2024-05-30T03:57:18.716Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c6f5f70f-fa0f-4180-b49d-c8b43bb610d2"}
2024-05-30T03:58:00.835Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "0c9e0fe8-c498-4da9-a09a-4ff0e6b8ea2f", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-2243 on 10.58.224.10:53: no such host"}
2024-05-30T03:58:05.926Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "1407b1ec-9991-4446-a4c9-67d83342906d", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-2243 on 10.58.224.10:53: no such host"}
2024-05-30T03:58:11.275Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "f1aa72c0-b2be-4eb0-aced-fa2618b4d9af", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-2243 on 10.58.224.10:53: no such host"}
2024-05-30T03:58:53.939Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "7eb14d85-89ac-44aa-83f8-f484ec813998", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-2243 on 10.58.224.10:53: no such host"}
2024-05-30T03:58:54.258Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "7adbedef-3cd9-4b40-b2f2-4a545f1820f6", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-2243 on 10.58.224.10:53: no such host"}
2024-05-30T03:58:59.259Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "91d0d7f9-1420-4181-9132-6b9e0b32a852", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-2243 on 10.58.224.10:53: no such host"}
2024-05-30T03:59:04.496Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "e9ee4457-f4ba-4f7d-8206-c888836e0e89", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-0: dial tcp 10.108.168.72:33062: connect: connection refused"}
2024-05-30T03:59:45.275Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "2b4de45b-cdfd-450f-aa5c-173759a8a773"}
2024-05-30T03:59:50.066Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "8fdbbb35-58e9-43b3-b037-a32f614b3269"}
2024-05-30T03:59:55.386Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "ace8677e-69c7-4548-86f2-a10e19609a24"}
2024-05-30T04:00:01.655Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "8735879f-f340-4a9c-bdb9-5c373ec08189"}
2024-05-30T04:00:07.254Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "35d3e5ba-ce94-4c97-bd63-202da34796e7"}
2024-05-30T04:00:08.862Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9d3518cc-d695-429f-8948-c7e6327c6386", "user": "monitor"}
2024-05-30T04:00:08.897Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9d3518cc-d695-429f-8948-c7e6327c6386", "user": "monitor"}
2024-05-30T04:00:08.905Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9d3518cc-d695-429f-8948-c7e6327c6386", "secret": "some-name-mysql-init", "user": "monitor"}
2024-05-30T04:00:08.968Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9d3518cc-d695-429f-8948-c7e6327c6386", "user": "monitor"}
2024-05-30T04:00:08.978Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9d3518cc-d695-429f-8948-c7e6327c6386", "user": "monitor"}
2024-05-30T04:00:09.072Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9d3518cc-d695-429f-8948-c7e6327c6386", "last-applied-secret": "4e3282050745046d5044e0fcb5f2bf140d7320a1256a79c8ca2184f38f4f3098"}
2024-05-30T04:00:12.932Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9d3518cc-d695-429f-8948-c7e6327c6386", "error": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / ", "errorVerbose": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-05-30T04:00:51.489Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "15407522-4fa7-41b7-a90d-0bcad85179d6", "user": "monitor"}
2024-05-30T04:00:55.886Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "15407522-4fa7-41b7-a90d-0bcad85179d6"}
2024-05-30T04:00:56.484Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "a760a130-5847-4932-a1f6-4001f9a6a76b", "user": "monitor"}
2024-05-30T04:00:56.744Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "a760a130-5847-4932-a1f6-4001f9a6a76b", "user": "monitor"}
2024-05-30T04:00:56.764Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "a760a130-5847-4932-a1f6-4001f9a6a76b", "last-applied-secret": "4e3282050745046d5044e0fcb5f2bf140d7320a1256a79c8ca2184f38f4f3098"}
2024-05-30T04:01:00.259Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "a760a130-5847-4932-a1f6-4001f9a6a76b"}
2024-05-30T04:01:05.754Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "94a0804d-4b17-4ad8-ab5a-066a1f72381f"}
2024-05-30T04:01:18.467Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "5368d66d-3a1a-47a8-a22b-35c6c8acd5ef"}
2024-05-30T04:01:24.031Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "4c4e93b5-806d-41f7-8802-04222ea24512"}
2024-05-30T04:01:25.724Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "b5f54592-e2ad-423c-82f6-dd9b3d44306c", "user": "operator"}
2024-05-30T04:01:25.759Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "b5f54592-e2ad-423c-82f6-dd9b3d44306c", "user": "operator"}
2024-05-30T04:01:25.767Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "b5f54592-e2ad-423c-82f6-dd9b3d44306c", "secret": "some-name-mysql-init", "user": "operator"}
2024-05-30T04:01:25.779Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "b5f54592-e2ad-423c-82f6-dd9b3d44306c", "user": "operator"}
2024-05-30T04:01:25.811Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "b5f54592-e2ad-423c-82f6-dd9b3d44306c", "user": "operator"}
2024-05-30T04:01:25.861Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "b5f54592-e2ad-423c-82f6-dd9b3d44306c", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-05-30T04:01:27.206Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "b5f54592-e2ad-423c-82f6-dd9b3d44306c", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-2243.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-2243.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-2243.svc.c' (using password: YES)\nERROR (line:569) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-2243.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-2243.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-2243.svc.c' (using password: YES)\nERROR (line:569) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-05-30T04:02:01.087Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9eb6b09a-8587-4102-9407-c1c0e694623c"}
2024-05-30T04:02:09.567Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "836f40d3-a80a-498a-a7f9-a61c9697fae9"}
2024-05-30T04:02:15.375Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c5b4d7cf-2aa7-4265-9c78-54ce2a2dc2a2"}
2024-05-30T04:02:20.658Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "1d5dfb12-f6ab-4c8e-bc3f-7843d52f60d4"}
2024-05-30T04:02:21.480Z	INFO	Created user secrets	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "secrets": "my-cluster-secrets-2"}
2024-05-30T04:02:21.492Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "root"}
2024-05-30T04:02:21.550Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "root"}
2024-05-30T04:02:21.561Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "secret": "some-name-mysql-init", "user": "root"}
2024-05-30T04:02:27.159Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69"}
2024-05-30T04:02:27.176Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "root"}
2024-05-30T04:02:27.231Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "root"}
2024-05-30T04:02:27.244Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "operator"}
2024-05-30T04:02:27.286Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "operator"}
2024-05-30T04:02:27.306Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "secret": "some-name-mysql-init", "user": "operator"}
2024-05-30T04:02:27.324Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "operator"}
2024-05-30T04:02:27.358Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "operator"}
2024-05-30T04:02:27.379Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "monitor"}
2024-05-30T04:02:27.413Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "monitor"}
2024-05-30T04:02:27.427Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "secret": "some-name-mysql-init", "user": "monitor"}
2024-05-30T04:02:27.474Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "monitor"}
2024-05-30T04:02:27.487Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "monitor"}
2024-05-30T04:02:27.635Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "xtrabackup"}
2024-05-30T04:02:27.670Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "xtrabackup"}
2024-05-30T04:02:27.716Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-05-30T04:02:27.752Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "xtrabackup"}
2024-05-30T04:02:27.784Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "xtrabackup"}
2024-05-30T04:02:27.796Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "replication"}
2024-05-30T04:02:27.832Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "replication"}
2024-05-30T04:02:27.850Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "secret": "some-name-mysql-init", "user": "replication"}
2024-05-30T04:02:27.873Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "replication"}
2024-05-30T04:02:27.906Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "replication"}
2024-05-30T04:02:27.906Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "proxyadmin"}
2024-05-30T04:02:27.954Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "proxyadmin"}
2024-05-30T04:02:27.968Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "user": "proxyadmin"}
2024-05-30T04:02:27.968Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "last-applied-secret": "c82746bfa019fb1380c987617e14b833c228d991286e43364ea5abeeccb221d1"}
2024-05-30T04:02:27.968Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "last-applied-secret": "c82746bfa019fb1380c987617e14b833c228d991286e43364ea5abeeccb221d1"}
2024-05-30T04:02:28.452Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "007cea8d-1842-428b-a685-102a339c2c69", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-05-30T04:03:26.853Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "36db466a-a737-418b-83a9-8324203b666c", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-2243 on 10.58.224.10:53: no such host"}
2024-05-30T04:03:32.897Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "8ae4060f-c886-4aec-8a9d-c000ba62c7c6", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp 10.108.169.45:33062: connect: connection refused"}
2024-05-30T04:04:20.351Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "692647bd-0f89-44fb-9cd8-86d6461f6d16", "err": "failed to connect to pod some-name-pxc-0: dial tcp 10.108.168.75:33062: connect: connection refused"}
2024-05-30T04:04:30.928Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "210ebd7c-9609-450c-9a62-5274a443e1b1", "primary name": "some-name-pxc-0.some-name-pxc.users-2243.svc.cluster.local"}
2024-05-30T04:04:41.372Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "a22c0551-9605-4e4e-8763-dff6380170f5", "primary name": "some-name-pxc-0.some-name-pxc.users-2243.svc.cluster.local"}
2024-05-30T04:04:58.142Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9a7d220f-e801-4abe-b9b2-a8898c74be3f", "user": "monitor"}
2024-05-30T04:04:58.637Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9a7d220f-e801-4abe-b9b2-a8898c74be3f", "user": "monitor"}
2024-05-30T04:04:58.682Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9a7d220f-e801-4abe-b9b2-a8898c74be3f", "last-applied-secret": "c82746bfa019fb1380c987617e14b833c228d991286e43364ea5abeeccb221d1"}
2024-05-30T04:05:02.161Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9a7d220f-e801-4abe-b9b2-a8898c74be3f"}
2024-05-30T04:05:05.865Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "821b97df-a6d5-469b-8a69-a1189ca4bc34"}
2024-05-30T04:05:07.723Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "44aa9eab-d880-427f-b6dc-face83d8fd39", "user": "operator"}
2024-05-30T04:05:07.754Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "44aa9eab-d880-427f-b6dc-face83d8fd39", "user": "operator"}
2024-05-30T04:05:07.765Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "44aa9eab-d880-427f-b6dc-face83d8fd39", "secret": "some-name-mysql-init", "user": "operator"}
2024-05-30T04:05:07.775Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "44aa9eab-d880-427f-b6dc-face83d8fd39", "user": "operator"}
2024-05-30T04:05:07.806Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "44aa9eab-d880-427f-b6dc-face83d8fd39", "user": "operator"}
2024-05-30T04:05:07.843Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "44aa9eab-d880-427f-b6dc-face83d8fd39", "last-applied-secret": "d3bfdd8f5de9a5a56ba120e88bab812f8335a60e801c14c73274a59b8cfdcd69"}
2024-05-30T04:05:09.195Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "44aa9eab-d880-427f-b6dc-face83d8fd39", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-2243.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-2243.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-2243.svc.c' (using password: YES)\nERROR (line:569) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-2243.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-2243.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-2243.svc.c' (using password: YES)\nERROR (line:569) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-05-30T04:05:35.792Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c11b11df-ce96-4c9d-84a7-17809c34b259"}
2024-05-30T04:05:44.746Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "8e58db64-c7e0-471a-b0fd-0e3425c53c46"}
2024-05-30T04:05:57.037Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c1e96d71-c00d-40e8-8b2c-060ca91f308a"}
2024-05-30T04:06:02.644Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "31508f2e-adef-4f40-aa0e-222f60a61a3c"}
2024-05-30T04:06:07.862Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "a1eb5c52-0bc9-4dde-b784-e63f0781573c"}
2024-05-30T04:06:13.680Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "6e29c73d-3423-4b8f-91ac-eb2c3b27d102"}
2024-05-30T04:06:18.983Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c35736f4-943c-41ca-b123-f6ae5cb20f30"}
2024-05-30T04:06:25.025Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "3b1f3b8e-1a33-41f4-88b8-159b5768b1b1"}
2024-05-30T04:06:30.267Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "7775eb13-dd57-4d5b-80b7-c0d84c953e7e"}
2024-05-30T04:06:34.867Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "38db352a-e6eb-4eb9-ab69-ce861849d2be"}
2024-05-30T04:06:40.173Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "a7d5bc41-3cab-4801-a6d1-57106d4f4f81"}
2024-05-30T04:06:45.552Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "c97862cd-05ab-4512-9491-84f176fe0a4f"}
2024-05-30T04:06:51.932Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "7b7007cb-aa6f-4215-a7c0-9b4d97ba9ecd"}
2024-05-30T04:06:57.518Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "cdc5c7ab-a0b0-4793-86b2-e998ba798c19"}
2024-05-30T04:07:02.963Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "6eab1838-7a1e-41a0-a244-da1063e1f12c"}
2024-05-30T04:07:08.382Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "7028b662-e6df-46ad-9152-1c40fa8f5205"}
2024-05-30T04:07:10.107Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "root"}
2024-05-30T04:07:10.171Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "root"}
2024-05-30T04:07:10.192Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "secret": "some-name-mysql-init", "user": "root"}
2024-05-30T04:07:15.823Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd"}
2024-05-30T04:07:15.835Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "root"}
2024-05-30T04:07:15.900Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "root"}
2024-05-30T04:07:15.947Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "monitor"}
2024-05-30T04:07:15.981Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "monitor"}
2024-05-30T04:07:15.990Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "secret": "some-name-mysql-init", "user": "monitor"}
2024-05-30T04:07:16.036Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "monitor"}
2024-05-30T04:07:16.048Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "monitor"}
2024-05-30T04:07:16.152Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "xtrabackup"}
2024-05-30T04:07:16.187Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "xtrabackup"}
2024-05-30T04:07:16.204Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-05-30T04:07:16.218Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "xtrabackup"}
2024-05-30T04:07:16.252Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "xtrabackup"}
2024-05-30T04:07:16.265Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "proxyadmin"}
2024-05-30T04:07:16.314Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "proxyadmin"}
2024-05-30T04:07:16.324Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "user": "proxyadmin"}
2024-05-30T04:07:16.324Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "last-applied-secret": "74c95093ad25aaa4a8d0e47613e7d53b28cf571029b69b85f0d00a530661e010"}
2024-05-30T04:07:16.324Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "last-applied-secret": "74c95093ad25aaa4a8d0e47613e7d53b28cf571029b69b85f0d00a530661e010"}
2024-05-30T04:07:16.598Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "fb3683f9-1dcf-4d10-b0fa-af0a588914bd", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-05-30T04:07:33.485Z	INFO	KubeAPIWarningLogger	.metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 049f8177-502e-4bb5-9496-e2d8ff047d3b
2024-05-30T04:09:03.596Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "5f38b560-0f7d-4e74-b2aa-f6aa1cf19068", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp 10.108.169.49:33062: connect: connection refused"}
2024-05-30T04:09:56.523Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "209f338c-55b5-4acb-97e4-f7ae096dea20", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-0: dial tcp 10.108.168.79:33062: connect: connection refused"}
2024-05-30T04:10:34.494Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "root"}
2024-05-30T04:10:34.546Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "root"}
2024-05-30T04:10:34.555Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "secret": "some-name-mysql-init", "user": "root"}
2024-05-30T04:10:34.567Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "root"}
2024-05-30T04:10:34.614Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "root"}
2024-05-30T04:10:34.626Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "operator"}
2024-05-30T04:10:34.660Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "operator"}
2024-05-30T04:10:34.670Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "secret": "some-name-mysql-init", "user": "operator"}
2024-05-30T04:10:34.685Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "operator"}
2024-05-30T04:10:34.719Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "operator"}
2024-05-30T04:10:34.731Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "monitor"}
2024-05-30T04:10:34.762Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "monitor"}
2024-05-30T04:10:34.770Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "secret": "some-name-mysql-init", "user": "monitor"}
2024-05-30T04:10:34.783Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "monitor"}
2024-05-30T04:10:34.871Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "xtrabackup"}
2024-05-30T04:10:34.900Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "xtrabackup"}
2024-05-30T04:10:34.908Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-05-30T04:10:34.918Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "xtrabackup"}
2024-05-30T04:10:34.950Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "xtrabackup"}
2024-05-30T04:10:34.961Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "replication"}
2024-05-30T04:10:34.992Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "replication"}
2024-05-30T04:10:35.009Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "secret": "some-name-mysql-init", "user": "replication"}
2024-05-30T04:10:35.027Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "replication"}
2024-05-30T04:10:35.058Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "user": "replication"}
2024-05-30T04:10:35.058Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-05-30T04:10:35.058Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "188f3502-1e54-417e-86a6-41a735d6f853", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-05-30T04:11:53.871Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "812c9f9b-44b4-4c6b-948a-386de9d9ecd8", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp 10.108.169.53:33062: i/o timeout"}
2024-05-30T04:12:09.741Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "398fdf26-1403-4aea-b937-84a4b3beb67e", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-2243 on 10.58.224.10:53: no such host"}
2024-05-30T04:12:10.105Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "2fd625f7-68ed-4e27-98ab-ae30c7013182", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-2243 on 10.58.224.10:53: no such host"}
2024-05-30T04:12:15.045Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "bf44d309-b230-430d-be67-23796d3fd1fa", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-2243 on 10.58.224.10:53: no such host"}
2024-05-30T04:12:52.075Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "f407d15e-1035-45f3-b4f7-f257190d1501", "user": "monitor"}
2024-05-30T04:12:52.382Z	INFO	Old password discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "f407d15e-1035-45f3-b4f7-f257190d1501", "user": "monitor"}
2024-05-30T04:12:52.405Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "f407d15e-1035-45f3-b4f7-f257190d1501", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-05-30T04:13:14.066Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9dce923d-9c0e-47d7-96c8-eb9b14d7c301", "user": "monitor"}
2024-05-30T04:13:14.097Z	INFO	Password updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9dce923d-9c0e-47d7-96c8-eb9b14d7c301", "user": "monitor"}
2024-05-30T04:13:14.106Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9dce923d-9c0e-47d7-96c8-eb9b14d7c301", "secret": "some-name-mysql-init", "user": "monitor"}
2024-05-30T04:13:14.115Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9dce923d-9c0e-47d7-96c8-eb9b14d7c301", "user": "monitor"}
2024-05-30T04:13:14.211Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "9dce923d-9c0e-47d7-96c8-eb9b14d7c301", "last-applied-secret": "81935ecf30ae523a6ee69582354f9488d47dbeb0f8c7c572af96870f2024284d"}
2024-05-30T04:14:24.132Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "f83226ee-daa1-48d1-89df-65bf053a06f4", "user": "monitor"}
2024-05-30T04:14:29.165Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "bc5ac1ad-8fd0-44de-b9a5-f7bab56542ed", "user": "monitor"}
2024-05-30T04:14:34.928Z	INFO	Password updated but old one not discarded	{"controller": "pxc-controller", "namespace": "users-2243", "name": "some-name", "reconcileID": "2bf57e55-c4a7-4777-a1e5-f34ce524365e", "user": "monitor"}
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.3/pkg/internal/controller/controller.go:222
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.3/pkg/internal/controller/controller.go:261
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.3/pkg/internal/controller/controller.go:324
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248
[mysql] 2024/05/30 04:03:32 packets.go:37: read tcp 10.108.168.70:51630->10.108.169.45:33062: read: connection reset by peer
[mysql] 2024/05/30 04:09:56 packets.go:37: read tcp 10.108.168.70:55102->10.108.168.79:33062: read: connection reset by peer
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
+ kubectl get pxc --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl patch pxc -n users-2243 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.5fjthNEtOK
++ mktemp
+ local LAST_ERR=/tmp/tmp.WGSkbHZm6b
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5fjthNEtOK
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.WGSkbHZm6b
+ rm /tmp/tmp.5fjthNEtOK /tmp/tmp.WGSkbHZm6b
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.oqsaYYwdUM
++ mktemp
+ local LAST_ERR=/tmp/tmp.uCXkFgtyMS
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.oqsaYYwdUM
No resources found
+ cat /tmp/tmp.uCXkFgtyMS
+ rm /tmp/tmp.oqsaYYwdUM /tmp/tmp.uCXkFgtyMS
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.dld7GyN3VR
++ mktemp
+ local LAST_ERR=/tmp/tmp.REWOUHkhIK
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.dld7GyN3VR
No resources found
+ cat /tmp/tmp.REWOUHkhIK
+ rm /tmp/tmp.dld7GyN3VR /tmp/tmp.REWOUHkhIK
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.94sy3K5UD9
++ mktemp
+ local LAST_ERR=/tmp/tmp.rCh6vDo5f4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.94sy3K5UD9
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.rCh6vDo5f4
+ rm /tmp/tmp.94sy3K5UD9 /tmp/tmp.rCh6vDo5f4
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace users-2243
+ rm -rf /tmp/tmp.0sjIU7TyM0
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.aSE2XT2BqJ
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

++ mktemp
+ local LAST_OUT=/tmp/tmp.wGYSDJ1ct4
+ local LAST_ERR=/tmp/tmp.vDNviHI8RI
++ mktemp
+ local exit_status=0
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.dz0OUnX3Fg
+ local exit_status=0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace users-2243
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator