Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/logs/users-5-7.log
WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1
+ create_infra users-25915
+ local ns=users-25915
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl patch pxc -n users-14485 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.xx3ealonx6
++ mktemp
+ local LAST_ERR=/tmp/tmp.ItlTHPmLcS
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xx3ealonx6
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.ItlTHPmLcS
+ rm /tmp/tmp.xx3ealonx6 /tmp/tmp.ItlTHPmLcS
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.detkMKsPqS
++ mktemp
+ local LAST_ERR=/tmp/tmp.rjQENswLpz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.detkMKsPqS
No resources found
+ cat /tmp/tmp.rjQENswLpz
+ rm /tmp/tmp.detkMKsPqS /tmp/tmp.rjQENswLpz
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.haZNVVlIVV
++ mktemp
+ local LAST_ERR=/tmp/tmp.tO32f8qhaT
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.haZNVVlIVV
No resources found
+ cat /tmp/tmp.tO32f8qhaT
+ rm /tmp/tmp.haZNVVlIVV /tmp/tmp.tO32f8qhaT
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ awk '{print$1}'
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.0ulphSEZPh
+ local LAST_OUT=/tmp/tmp.NSdZPhtKAZ
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.Vogrhvz8zj
+ local exit_status=0
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.W2ucatB24j
+ local exit_status=0
+ xargs kubectl delete ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.NSdZPhtKAZ
+ cat /tmp/tmp.W2ucatB24j
+ rm /tmp/tmp.NSdZPhtKAZ /tmp/tmp.W2ucatB24j
+ return 0
namespace "users-14485" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0ulphSEZPh
namespace "pxc-operator" deleted
+ cat /tmp/tmp.Vogrhvz8zj
+ rm /tmp/tmp.0ulphSEZPh /tmp/tmp.Vogrhvz8zj
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'namespace/pxc-operator - '
namespace/pxc-operator - + set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.YLSkztNW8B
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZrQI8n3GIA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.YLSkztNW8B
namespace/pxc-operator created
+ cat /tmp/tmp.ZrQI8n3GIA
+ rm /tmp/tmp.YLSkztNW8B /tmp/tmp.ZrQI8n3GIA
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.53Ipv2T0LT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AEPsaTWmV0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.53Ipv2T0LT
++ cat /tmp/tmp.AEPsaTWmV0
++ rm /tmp/tmp.53Ipv2T0LT /tmp/tmp.AEPsaTWmV0
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1588-5c1f635c-6-cluster4 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.4bAmkilZFN
++ mktemp
+ local LAST_ERR=/tmp/tmp.jqeNMeaIHZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1588-5c1f635c-6-cluster4 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.4bAmkilZFN
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1588-5c1f635c-6-cluster4" modified.
+ cat /tmp/tmp.jqeNMeaIHZ
+ rm /tmp/tmp.4bAmkilZFN /tmp/tmp.jqeNMeaIHZ
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.2Tubt2xo6G
++ mktemp
+ local LAST_ERR=/tmp/tmp.IcxllTCUuj
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2Tubt2xo6G
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.IcxllTCUuj
+ rm /tmp/tmp.2Tubt2xo6G /tmp/tmp.IcxllTCUuj
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/deploy/cw-rbac.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.9tOdoOekYk
++ mktemp
+ local LAST_ERR=/tmp/tmp.FPMf3iLjZf
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.9tOdoOekYk
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.FPMf3iLjZf
+ rm /tmp/tmp.9tOdoOekYk /tmp/tmp.FPMf3iLjZf
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1588-5c1f635c^'
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/deploy/cw-operator.yaml
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' -
+ kubectl_bin apply -f -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.JbFwEiBalt
++ mktemp
+ local LAST_ERR=/tmp/tmp.6fqUceqtdz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.JbFwEiBalt
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.6fqUceqtdz
+ rm /tmp/tmp.JbFwEiBalt /tmp/tmp.6fqUceqtdz
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.pKCv99FFoT
++ mktemp
+ local LAST_ERR=/tmp/tmp.KYlfc5UhrV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.pKCv99FFoT
pod/percona-xtradb-cluster-operator-66688f478-8czhs condition met
+ cat /tmp/tmp.KYlfc5UhrV
+ rm /tmp/tmp.pKCv99FFoT /tmp/tmp.KYlfc5UhrV
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GCDqXRAkgD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0OOSfznzVV
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GCDqXRAkgD
++ cat /tmp/tmp.0OOSfznzVV
++ rm /tmp/tmp.GCDqXRAkgD /tmp/tmp.0OOSfznzVV
++ return 0
+ wait_pod percona-xtradb-cluster-operator-66688f478-8czhs 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-66688f478-8czhs
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-66688f478-8czhs
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-66688f478-8czhs condition met
percona-xtradb-cluster-operator-66688f478-8czhs.Ok
+ sleep 3
+ create_namespace users-25915
+ local namespace=users-25915
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces users-25915'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces users-25915
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace users-25915
+ xargs kubectl delete ns
++ mktemp
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.402fM6G6Js
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.2amTbfxwRO
+ local exit_status=0
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.gpC8oUiN5t
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-25915
+ local LAST_ERR=/tmp/tmp.gkWEqCzbUm
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gpC8oUiN5t
+ cat /tmp/tmp.gkWEqCzbUm
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ rm /tmp/tmp.gpC8oUiN5t /tmp/tmp.gkWEqCzbUm
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-25915
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-25915
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.402fM6G6Js
+ cat /tmp/tmp.2amTbfxwRO
Error from server (NotFound): namespaces "users-25915" not found
+ rm /tmp/tmp.402fM6G6Js /tmp/tmp.2amTbfxwRO
+ return 1
+ :
+ wait_for_delete namespace/users-25915
+ local res=namespace/users-25915
+ echo -n 'namespace/users-25915 - '
namespace/users-25915 - + set +o xtrace
Error from server (NotFound): namespaces "users-25915" not found
+ desc 'create namespace users-25915'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace users-25915
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace users-25915
++ mktemp
+ local LAST_OUT=/tmp/tmp.mx6khnEoO7
++ mktemp
+ local LAST_ERR=/tmp/tmp.Mm2NMnZ2cE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace users-25915
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.mx6khnEoO7
namespace/users-25915 created
+ cat /tmp/tmp.Mm2NMnZ2cE
+ rm /tmp/tmp.mx6khnEoO7 /tmp/tmp.Mm2NMnZ2cE
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HYcxqEbrMu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JaHdB6flbR
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.HYcxqEbrMu
++ cat /tmp/tmp.JaHdB6flbR
++ rm /tmp/tmp.HYcxqEbrMu /tmp/tmp.JaHdB6flbR
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1588-5c1f635c-6-cluster4 --namespace=users-25915
++ mktemp
+ local LAST_OUT=/tmp/tmp.RjbOeF4Uw8
++ mktemp
+ local LAST_ERR=/tmp/tmp.VGqXg2IBGO
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1588-5c1f635c-6-cluster4 --namespace=users-25915
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.RjbOeF4Uw8
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1588-5c1f635c-6-cluster4" modified.
+ cat /tmp/tmp.VGqXg2IBGO
+ rm /tmp/tmp.RjbOeF4Uw8 /tmp/tmp.VGqXg2IBGO
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.u7Osbm1ZgC
++ mktemp
+ local LAST_ERR=/tmp/tmp.lUHJkgcgvD
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.u7Osbm1ZgC
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.lUHJkgcgvD
+ rm /tmp/tmp.u7Osbm1ZgC /tmp/tmp.lUHJkgcgvD
+ return 0
+ desc 'create PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC cluster
-----------------------------------------------------------------------------------

+ newpass=test-password
++ echo -n test-password
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZA==
+ cluster=some-name
+ spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/some-name.yml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/some-name.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.bZ9tLM5ALQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.4CzVwjpyXh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.bZ9tLM5ALQ
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.4CzVwjpyXh
+ rm /tmp/tmp.bZ9tLM5ALQ /tmp/tmp.4CzVwjpyXh
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1588-5c1f635c#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ local LAST_OUT=/tmp/tmp.xVlRfdXvqR
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-25915~
++ mktemp
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ local LAST_ERR=/tmp/tmp.LC8kP2dDNi
+ local exit_status=0
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xVlRfdXvqR
deployment.apps/pxc-client created
+ cat /tmp/tmp.LC8kP2dDNi
+ rm /tmp/tmp.xVlRfdXvqR /tmp/tmp.LC8kP2dDNi
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ some-name == \d\e\m\a\n\d\-\b\a\c\k\u\p ]]
+ [[ some-name == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/conf/some-name.yml
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ local LAST_OUT=/tmp/tmp.JfjnpjZ2Za
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-25915~
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1588-5c1f635c#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ local LAST_ERR=/tmp/tmp.NyK1f0GoJP
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.JfjnpjZ2Za
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.NyK1f0GoJP
+ rm /tmp/tmp.JfjnpjZ2Za /tmp/tmp.NyK1f0GoJP
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy some-name
++ local target_cluster=some-name
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SmKcJdQSjg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cMGIukx4eT
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.SmKcJdQSjg
+++ cat /tmp/tmp.cMGIukx4eT
+++ rm /tmp/tmp.SmKcJdQSjg /tmp/tmp.cMGIukx4eT
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kUkf5VilEi
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cXXdUZSzZm
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.kUkf5VilEi
+++ cat /tmp/tmp.cXXdUZSzZm
+++ rm /tmp/tmp.kUkf5VilEi /tmp/tmp.cXXdUZSzZm
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo some-name-proxysql
++ return
+ local proxy=some-name-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-25915
++ mktemp
+ local LAST_OUT=/tmp/tmp.w0fyaGTgj3
++ mktemp
+ local LAST_ERR=/tmp/tmp.r780tutkZp
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-25915
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-25915
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-25915
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.w0fyaGTgj3
+ cat /tmp/tmp.r780tutkZp
error: no matching resources found
+ rm /tmp/tmp.w0fyaGTgj3 /tmp/tmp.r780tutkZp
+ return 1
+ true
+ wait_for_running some-name-proxysql 1
+ local name=some-name-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-proxysql-0 480
+ local pod=some-name-proxysql-0
+ local max_retry=480
+ local ns=
++ echo some-name-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
pod/some-name-proxysql-0 condition met
some-name-proxysql-0.Ok
+ wait_for_running some-name-pxc 3
+ local name=some-name-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
some-name-pxc-0.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-1 480
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
some-name-pxc-1.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-2 480
+ local pod=some-name-pxc-2
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-2 condition met
some-name-pxc-2.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
++ is_keyring_plugin_in_use some-name
++ local cluster=some-name
++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xQ00aUkyGD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XqqN4hWVrU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xQ00aUkyGD
++ cat /tmp/tmp.XqqN4hWVrU
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.xQ00aUkyGD /tmp/tmp.XqqN4hWVrU
++ return 0
+ [[ -n '' ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-proxysql -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h some-name-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FbE1ayLVjA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.D3Jzvn601N
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FbE1ayLVjA
++ cat /tmp/tmp.D3Jzvn601N
++ rm /tmp/tmp.FbE1ayLVjA /tmp/tmp.D3Jzvn601N
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-proxysql -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h some-name-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QJc0hyHfBN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nAgtPVd0Bj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.QJc0hyHfBN
++ cat /tmp/tmp.nAgtPVd0Bj
++ rm /tmp/tmp.QJc0hyHfBN /tmp/tmp.nAgtPVd0Bj
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QTLGgtdg7n
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N70UE50QO8
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.QTLGgtdg7n
++ cat /tmp/tmp.N70UE50QO8
++ rm /tmp/tmp.QTLGgtdg7n /tmp/tmp.N70UE50QO8
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1.sql /tmp/tmp.oHgClg8g3G/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZmxxUoLZqZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MVjFTdbUUi
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZmxxUoLZqZ
++ cat /tmp/tmp.MVjFTdbUUi
++ rm /tmp/tmp.ZmxxUoLZqZ /tmp/tmp.MVjFTdbUUi
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1.sql /tmp/tmp.oHgClg8g3G/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.r5cGkOIZMA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jh52hR92hZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.r5cGkOIZMA
++ cat /tmp/tmp.jh52hR92hZ
++ rm /tmp/tmp.r5cGkOIZMA /tmp/tmp.jh52hR92hZ
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-1.sql /tmp/tmp.oHgClg8g3G/select-1.sql
++ is_keyring_plugin_in_use some-name
++ local cluster=some-name
++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vwksF0BsIJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eQPCZo3XeA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vwksF0BsIJ
++ cat /tmp/tmp.eQPCZo3XeA
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.vwksF0BsIJ /tmp/tmp.eQPCZo3XeA
++ return 0
+ '[' '' ']'
+ desc 'test root'
+ set +o xtrace


-----------------------------------------------------------------------------------
test root
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets root dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=root
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.gME5A0YO3p
++ mktemp
+ local LAST_ERR=/tmp/tmp.OqOQNSB0yl
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gME5A0YO3p
secret/my-cluster-secrets patched
+ cat /tmp/tmp.OqOQNSB0yl
+ rm /tmp/tmp.gME5A0YO3p /tmp/tmp.OqOQNSB0yl
+ return 0
+ sleep 15
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8qmfwceykW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.w9hnR7iXhg
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8qmfwceykW
++ cat /tmp/tmp.w9hnR7iXhg
++ rm /tmp/tmp.8qmfwceykW /tmp/tmp.w9hnR7iXhg
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql /tmp/tmp.oHgClg8g3G/select-4.sql
+ desc 'test proxyadmin'
+ set +o xtrace


-----------------------------------------------------------------------------------
test proxyadmin
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.7yZv1ooGKT
++ mktemp
+ local LAST_ERR=/tmp/tmp.grrEW9dhKd
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7yZv1ooGKT
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.grrEW9dhKd
+ rm /tmp/tmp.7yZv1ooGKT /tmp/tmp.grrEW9dhKd
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cKNoCTTMRx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4HjGppCVKD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cKNoCTTMRx
++ cat /tmp/tmp.4HjGppCVKD
++ rm /tmp/tmp.cKNoCTTMRx /tmp/tmp.4HjGppCVKD
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XoY9ROdyHl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wEblAfvxev
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XoY9ROdyHl
++ cat /tmp/tmp.wEblAfvxev
++ rm /tmp/tmp.XoY9ROdyHl /tmp/tmp.wEblAfvxev
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.w9COXRAkwj
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.KMR8OZebxq
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.w9COXRAkwj
+++++ cat /tmp/tmp.KMR8OZebxq
+++++ rm /tmp/tmp.w9COXRAkwj /tmp/tmp.KMR8OZebxq
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.9pl8F7XPEL
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.oLflyBJzak
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.9pl8F7XPEL
+++++ cat /tmp/tmp.oLflyBJzak
+++++ rm /tmp/tmp.9pl8F7XPEL /tmp/tmp.oLflyBJzak
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OMb0HwrMuk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.67sDHkv3AQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.OMb0HwrMuk
++ cat /tmp/tmp.67sDHkv3AQ
++ rm /tmp/tmp.OMb0HwrMuk /tmp/tmp.67sDHkv3AQ
++ return 0
+ [[ 3 == \3 ]]
+ patch_secret my-cluster-secrets proxyadmin dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=proxyadmin
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.GCtGeQOnZp
++ mktemp
+ local LAST_ERR=/tmp/tmp.3JdTzYYyZz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.GCtGeQOnZp
secret/my-cluster-secrets patched
+ cat /tmp/tmp.3JdTzYYyZz
+ rm /tmp/tmp.GCtGeQOnZp /tmp/tmp.3JdTzYYyZz
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GpVnSIFv65
+++ mktemp
++ local LAST_ERR=/tmp/tmp.628OvabeMF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GpVnSIFv65
++ cat /tmp/tmp.628OvabeMF
++ rm /tmp/tmp.GpVnSIFv65 /tmp/tmp.628OvabeMF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NxTD1ZVEMO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1utyJs1fb0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NxTD1ZVEMO
++ cat /tmp/tmp.1utyJs1fb0
++ rm /tmp/tmp.NxTD1ZVEMO /tmp/tmp.1utyJs1fb0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KfnTxZeGwJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tqvsyOvzSK
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KfnTxZeGwJ
++ cat /tmp/tmp.tqvsyOvzSK
++ rm /tmp/tmp.KfnTxZeGwJ /tmp/tmp.tqvsyOvzSK
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hWIprpMPEh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UiagRWKJ1s
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hWIprpMPEh
++ cat /tmp/tmp.UiagRWKJ1s
++ rm /tmp/tmp.hWIprpMPEh /tmp/tmp.UiagRWKJ1s
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.0zjlvqTk28
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.EuojbXtnOu
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.0zjlvqTk28
+++++ cat /tmp/tmp.EuojbXtnOu
+++++ rm /tmp/tmp.0zjlvqTk28 /tmp/tmp.EuojbXtnOu
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.6WrIM9Se19
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.7m8KA1mZtZ
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.6WrIM9Se19
+++++ cat /tmp/tmp.7m8KA1mZtZ
+++++ rm /tmp/tmp.6WrIM9Se19 /tmp/tmp.7m8KA1mZtZ
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nGp7ZovW3w
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J3ainUgtqD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nGp7ZovW3w
++ cat /tmp/tmp.J3ainUgtqD
++ rm /tmp/tmp.nGp7ZovW3w /tmp/tmp.J3ainUgtqD
++ return 0
+ [[ 3 == \3 ]]
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-0
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-0
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2.sql /tmp/tmp.oHgClg8g3G/select-2.sql
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-1
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-1
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2.sql /tmp/tmp.oHgClg8g3G/select-2.sql
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-2
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-2
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-2.sql /tmp/tmp.oHgClg8g3G/select-2.sql
+ desc 'test xtrabackup'
+ set +o xtrace


-----------------------------------------------------------------------------------
test xtrabackup
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.nXZIdRABQi
++ mktemp
+ local LAST_ERR=/tmp/tmp.OrvnR0n8Cl
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nXZIdRABQi
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.OrvnR0n8Cl
+ rm /tmp/tmp.nXZIdRABQi /tmp/tmp.OrvnR0n8Cl
+ return 0
+ patch_secret my-cluster-secrets xtrabackup dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=xtrabackup
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.vBdYTTJOZt
++ mktemp
+ local LAST_ERR=/tmp/tmp.n0Zg2Jthgh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vBdYTTJOZt
secret/my-cluster-secrets patched
+ cat /tmp/tmp.n0Zg2Jthgh
+ rm /tmp/tmp.vBdYTTJOZt /tmp/tmp.n0Zg2Jthgh
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YrfZRjy3se
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nGi3v1hEWN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YrfZRjy3se
++ cat /tmp/tmp.nGi3v1hEWN
++ rm /tmp/tmp.YrfZRjy3se /tmp/tmp.nGi3v1hEWN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xqRyoXqYaJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.U185EBREl1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xqRyoXqYaJ
++ cat /tmp/tmp.U185EBREl1
++ rm /tmp/tmp.xqRyoXqYaJ /tmp/tmp.U185EBREl1
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HM0ESG7QMf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DHDxFrryyc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.HM0ESG7QMf
++ cat /tmp/tmp.DHDxFrryyc
++ rm /tmp/tmp.HM0ESG7QMf /tmp/tmp.DHDxFrryyc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.j02nBItP0I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.burvb5ERuQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.j02nBItP0I
++ cat /tmp/tmp.burvb5ERuQ
++ rm /tmp/tmp.j02nBItP0I /tmp/tmp.burvb5ERuQ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7jVbYRQXXV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6gH4sHfRLg
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7jVbYRQXXV
++ cat /tmp/tmp.6gH4sHfRLg
++ rm /tmp/tmp.7jVbYRQXXV /tmp/tmp.6gH4sHfRLg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hK0s3bpY9C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VLhGTGfuNc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hK0s3bpY9C
++ cat /tmp/tmp.VLhGTGfuNc
++ rm /tmp/tmp.hK0s3bpY9C /tmp/tmp.VLhGTGfuNc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k3TmKwguO7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4sU4fyxayt
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.k3TmKwguO7
++ cat /tmp/tmp.4sU4fyxayt
++ rm /tmp/tmp.k3TmKwguO7 /tmp/tmp.4sU4fyxayt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fRaiuJQMSE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xU6JUcs7uO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.fRaiuJQMSE
++ cat /tmp/tmp.xU6JUcs7uO
++ rm /tmp/tmp.fRaiuJQMSE /tmp/tmp.xU6JUcs7uO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mk9jBt84RQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uE10PPuY5r
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.mk9jBt84RQ
++ cat /tmp/tmp.uE10PPuY5r
++ rm /tmp/tmp.mk9jBt84RQ /tmp/tmp.uE10PPuY5r
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8dG36n6gls
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KMM3Eqwjk6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8dG36n6gls
++ cat /tmp/tmp.KMM3Eqwjk6
++ rm /tmp/tmp.8dG36n6gls /tmp/tmp.KMM3Eqwjk6
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.efbmEQuO8Z
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.RLfVktZIlt
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.efbmEQuO8Z
+++++ cat /tmp/tmp.RLfVktZIlt
+++++ rm /tmp/tmp.efbmEQuO8Z /tmp/tmp.RLfVktZIlt
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.A0Jip8kCAS
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.2TnUGbRC51
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.A0Jip8kCAS
+++++ cat /tmp/tmp.2TnUGbRC51
+++++ rm /tmp/tmp.A0Jip8kCAS /tmp/tmp.2TnUGbRC51
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.16MTtQ10Bk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HlPNUGyXFt
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.16MTtQ10Bk
++ cat /tmp/tmp.HlPNUGyXFt
++ rm /tmp/tmp.16MTtQ10Bk /tmp/tmp.HlPNUGyXFt
++ return 0
+ [[ 2 == \2 ]]
+ compare_mysql_cmd_local select-3 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 '' pxc
+ local command_id=select-3
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\'''
+ local pod=some-name-pxc-0
+ local postfix=
+ local container_name=pxc
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql_local 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 pxc
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\'''
+ local pod=some-name-pxc-0
+ local container_name=pxc
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-3.sql /tmp/tmp.oHgClg8g3G/select-3.sql
+ desc 'test monitor'
+ set +o xtrace


-----------------------------------------------------------------------------------
test monitor
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=monitor
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.UrV3d63qgQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.iaoxWpJXAG
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UrV3d63qgQ
secret/my-cluster-secrets patched
+ cat /tmp/tmp.iaoxWpJXAG
+ rm /tmp/tmp.UrV3d63qgQ /tmp/tmp.iaoxWpJXAG
+ return 0
+ wait_for_password_propagation my-cluster-secrets monitor
+ local secret=my-cluster-secrets
+ local user=monitor
+ local max_retry=240
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oiqUw1RtfQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2X2KQLD6vk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oiqUw1RtfQ
++ cat /tmp/tmp.2X2KQLD6vk
++ rm /tmp/tmp.oiqUw1RtfQ /tmp/tmp.2X2KQLD6vk
++ return 0
+ local root_pass=test-password
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ echo 'Skipping dual password feature doesn'\''t work for 5.7. PXC 5.7 doesn'\''t support it!'
Skipping dual password feature doesn't work for 5.7. PXC 5.7 doesn't support it!
+ return
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RQ13xk2DpN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fXLvdUyYby
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RQ13xk2DpN
++ cat /tmp/tmp.fXLvdUyYby
++ rm /tmp/tmp.RQ13xk2DpN /tmp/tmp.fXLvdUyYby
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.srxX7vPi2q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SsZHNf7iTr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.srxX7vPi2q
++ cat /tmp/tmp.SsZHNf7iTr
++ rm /tmp/tmp.srxX7vPi2q /tmp/tmp.SsZHNf7iTr
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vas6Gioaja
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o1XSQELOYy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vas6Gioaja
++ cat /tmp/tmp.o1XSQELOYy
++ rm /tmp/tmp.vas6Gioaja /tmp/tmp.o1XSQELOYy
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TZlmQk4UJf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N8QDWd5pTc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TZlmQk4UJf
++ cat /tmp/tmp.N8QDWd5pTc
++ rm /tmp/tmp.TZlmQk4UJf /tmp/tmp.N8QDWd5pTc
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.QK2pxE4TIi
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.nMorCWF8xC
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.QK2pxE4TIi
+++++ cat /tmp/tmp.nMorCWF8xC
+++++ rm /tmp/tmp.QK2pxE4TIi /tmp/tmp.nMorCWF8xC
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.bNVYuk40Nx
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.4cQCi2qMU3
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.bNVYuk40Nx
+++++ cat /tmp/tmp.4cQCi2qMU3
+++++ rm /tmp/tmp.bNVYuk40Nx /tmp/tmp.4cQCi2qMU3
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Hbfv9mnO5g
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DxI1jorXng
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Hbfv9mnO5g
++ cat /tmp/tmp.DxI1jorXng
++ rm /tmp/tmp.Hbfv9mnO5g /tmp/tmp.DxI1jorXng
++ return 0
+ [[ 2 == \2 ]]
+ sleep 10
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8HJrQa6suc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZH6EC9Uk7Q
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8HJrQa6suc
++ cat /tmp/tmp.ZH6EC9Uk7Q
++ rm /tmp/tmp.8HJrQa6suc /tmp/tmp.ZH6EC9Uk7Q
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql /tmp/tmp.oHgClg8g3G/select-4.sql
+ desc 'test operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
test operator
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets operator dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=operator
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.5vpHQfknKp
++ mktemp
+ local LAST_ERR=/tmp/tmp.r6lJFVmUTt
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5vpHQfknKp
secret/my-cluster-secrets patched
+ cat /tmp/tmp.r6lJFVmUTt
+ rm /tmp/tmp.5vpHQfknKp /tmp/tmp.r6lJFVmUTt
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yhnmxM3lpR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WQ7Q61feKz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.yhnmxM3lpR
++ cat /tmp/tmp.WQ7Q61feKz
++ rm /tmp/tmp.yhnmxM3lpR /tmp/tmp.WQ7Q61feKz
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3LcgqJu2lT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6aY2crH68y
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3LcgqJu2lT
++ cat /tmp/tmp.6aY2crH68y
++ rm /tmp/tmp.3LcgqJu2lT /tmp/tmp.6aY2crH68y
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WKCPxj9Xq4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PoMfMiXF00
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WKCPxj9Xq4
++ cat /tmp/tmp.PoMfMiXF00
++ rm /tmp/tmp.WKCPxj9Xq4 /tmp/tmp.PoMfMiXF00
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.f0BcnZ2FKG
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.nta4hKDtqu
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.f0BcnZ2FKG
+++++ cat /tmp/tmp.nta4hKDtqu
+++++ rm /tmp/tmp.f0BcnZ2FKG /tmp/tmp.nta4hKDtqu
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.UZdHmZGPIi
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.CppdhI8YnL
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.UZdHmZGPIi
+++++ cat /tmp/tmp.CppdhI8YnL
+++++ rm /tmp/tmp.UZdHmZGPIi /tmp/tmp.CppdhI8YnL
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L3nDcMThTa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.P1qo6EvUyx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.L3nDcMThTa
++ cat /tmp/tmp.P1qo6EvUyx
++ rm /tmp/tmp.L3nDcMThTa /tmp/tmp.P1qo6EvUyx
++ return 0
+ [[ 2 == \2 ]]
+ sleep 10
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rkAozLuqpZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5Bj0EHgMzD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rkAozLuqpZ
++ cat /tmp/tmp.5Bj0EHgMzD
++ rm /tmp/tmp.rkAozLuqpZ /tmp/tmp.5Bj0EHgMzD
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql /tmp/tmp.oHgClg8g3G/select-4.sql
+ desc 'change secret name'
+ set +o xtrace


-----------------------------------------------------------------------------------
change secret name
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.u9XpYdZ0DL
++ mktemp
+ local LAST_ERR=/tmp/tmp.kFCRys2Td6
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.u9XpYdZ0DL
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.kFCRys2Td6
+ rm /tmp/tmp.u9XpYdZ0DL /tmp/tmp.kFCRys2Td6
+ return 0
+ sleep 30
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a2kfUyO38c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yIFQ11JcSG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.a2kfUyO38c
++ cat /tmp/tmp.yIFQ11JcSG
++ rm /tmp/tmp.a2kfUyO38c /tmp/tmp.yIFQ11JcSG
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TECNZELer5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4VOtoS4stv
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TECNZELer5
++ cat /tmp/tmp.4VOtoS4stv
++ rm /tmp/tmp.TECNZELer5 /tmp/tmp.4VOtoS4stv
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tRJ0IrvyhL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1CHsgaRmLL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tRJ0IrvyhL
++ cat /tmp/tmp.1CHsgaRmLL
++ rm /tmp/tmp.tRJ0IrvyhL /tmp/tmp.1CHsgaRmLL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8dZxxni1ty
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8HqnvPXWEp
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8dZxxni1ty
++ cat /tmp/tmp.8HqnvPXWEp
++ rm /tmp/tmp.8dZxxni1ty /tmp/tmp.8HqnvPXWEp
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U7C11MZon8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xUoZTknFmw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.U7C11MZon8
++ cat /tmp/tmp.xUoZTknFmw
++ rm /tmp/tmp.U7C11MZon8 /tmp/tmp.xUoZTknFmw
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p7jsc6kEEf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aZbn5BWtua
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.p7jsc6kEEf
++ cat /tmp/tmp.aZbn5BWtua
++ rm /tmp/tmp.p7jsc6kEEf /tmp/tmp.aZbn5BWtua
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NJxL4Twn8c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dy58DeVry5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NJxL4Twn8c
++ cat /tmp/tmp.dy58DeVry5
++ rm /tmp/tmp.NJxL4Twn8c /tmp/tmp.dy58DeVry5
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oCd8HO80OV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bKHF2Kw2t0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oCd8HO80OV
++ cat /tmp/tmp.bKHF2Kw2t0
++ rm /tmp/tmp.oCd8HO80OV /tmp/tmp.bKHF2Kw2t0
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jh2iAgZnTw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6dVdUdcTqJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.jh2iAgZnTw
++ cat /tmp/tmp.6dVdUdcTqJ
++ rm /tmp/tmp.jh2iAgZnTw /tmp/tmp.6dVdUdcTqJ
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.ofsOfTje7E
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.vuoX36zwpi
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.ofsOfTje7E
+++++ cat /tmp/tmp.vuoX36zwpi
+++++ rm /tmp/tmp.ofsOfTje7E /tmp/tmp.vuoX36zwpi
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.F5yQkURJmk
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.QiDAGj7Jyt
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.F5yQkURJmk
+++++ cat /tmp/tmp.QiDAGj7Jyt
+++++ rm /tmp/tmp.F5yQkURJmk /tmp/tmp.QiDAGj7Jyt
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0bWt4DcNmK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.w0EbmWruOW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0bWt4DcNmK
++ cat /tmp/tmp.w0EbmWruOW
++ rm /tmp/tmp.0bWt4DcNmK /tmp/tmp.w0EbmWruOW
++ return 0
+ [[ 2 == \2 ]]
+ desc 'test new operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
test new operator
-----------------------------------------------------------------------------------

+ newpass=test-password2
++ echo -n test-password2
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZDI=
+ patch_secret my-cluster-secrets-2 operator dGVzdC1wYXNzd29yZDI=
+ local secret=my-cluster-secrets-2
+ local key=operator
+ local value=dGVzdC1wYXNzd29yZDI=
+ kubectl_bin patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.VkEJLA6BfK
++ mktemp
+ local LAST_ERR=/tmp/tmp.33w7wUjCUU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.VkEJLA6BfK
secret/my-cluster-secrets-2 patched
+ cat /tmp/tmp.33w7wUjCUU
+ rm /tmp/tmp.VkEJLA6BfK /tmp/tmp.33w7wUjCUU
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zz3MP03ZhK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.US8AM3EGUI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zz3MP03ZhK
++ cat /tmp/tmp.US8AM3EGUI
++ rm /tmp/tmp.zz3MP03ZhK /tmp/tmp.US8AM3EGUI
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lATbj1MdNK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uU1JqJs5SJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.lATbj1MdNK
++ cat /tmp/tmp.uU1JqJs5SJ
++ rm /tmp/tmp.lATbj1MdNK /tmp/tmp.uU1JqJs5SJ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RNvIvnvuAp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dRsOFht5ow
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RNvIvnvuAp
++ cat /tmp/tmp.dRsOFht5ow
++ rm /tmp/tmp.RNvIvnvuAp /tmp/tmp.dRsOFht5ow
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Vpv7ND9gbX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jjyjJWJuUk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Vpv7ND9gbX
++ cat /tmp/tmp.jjyjJWJuUk
++ rm /tmp/tmp.Vpv7ND9gbX /tmp/tmp.jjyjJWJuUk
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.y6hJpaYQbW
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.X6fcUa15VH
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.y6hJpaYQbW
+++++ cat /tmp/tmp.X6fcUa15VH
+++++ rm /tmp/tmp.y6hJpaYQbW /tmp/tmp.X6fcUa15VH
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.NjKPaJ0kgj
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.92jnMDw35M
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.NjKPaJ0kgj
+++++ cat /tmp/tmp.92jnMDw35M
+++++ rm /tmp/tmp.NjKPaJ0kgj /tmp/tmp.92jnMDw35M
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VJza3FJkcl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qOmln2Omm6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VJza3FJkcl
++ cat /tmp/tmp.qOmln2Omm6
++ rm /tmp/tmp.VJza3FJkcl /tmp/tmp.qOmln2Omm6
++ return 0
+ [[ 2 == \2 ]]
+ sleep 20
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FdptAAI1wJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4zu78bdYTM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FdptAAI1wJ
++ cat /tmp/tmp.4zu78bdYTM
++ rm /tmp/tmp.FdptAAI1wJ /tmp/tmp.4zu78bdYTM
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql /tmp/tmp.oHgClg8g3G/select-4.sql
++ getSecretData my-cluster-secrets-2 root
++ local secretName=my-cluster-secrets-2
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets-2 '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.URHwiLKtcb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UP0jSAWeST
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets-2 '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.URHwiLKtcb
++ cat /tmp/tmp.UP0jSAWeST
++ rm /tmp/tmp.URHwiLKtcb /tmp/tmp.UP0jSAWeST
++ return 0
+ newpass='BK)>PN>>h&L(s0GYv<'
+ desc 'test new users sync'
+ set +o xtrace


-----------------------------------------------------------------------------------
test new users sync
-----------------------------------------------------------------------------------

+ run_mysql 'CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\''BK)>PN>>h&L(s0GYv<'\'';' '-h some-name-pxc -uroot -p'\''BK)>PN>>h&L(s0GYv<'\'''
+ local 'command=CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\''BK)>PN>>h&L(s0GYv<'\'';'
+ local 'uri=-h some-name-pxc -uroot -p'\''BK)>PN>>h&L(s0GYv<'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.03j0saVV6x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VqXoXlcZ01
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.03j0saVV6x
++ cat /tmp/tmp.VqXoXlcZ01
++ rm /tmp/tmp.03j0saVV6x /tmp/tmp.VqXoXlcZ01
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ sleep 40
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\''BK)>PN>>h&L(s0GYv<'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -utestsync -p'\''BK)>PN>>h&L(s0GYv<'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\''BK)>PN>>h&L(s0GYv<'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -utestsync -p'\''BK)>PN>>h&L(s0GYv<'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PG0inJZleq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yMSk7L17ao
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PG0inJZleq
++ cat /tmp/tmp.yMSk7L17ao
++ rm /tmp/tmp.PG0inJZleq /tmp/tmp.yMSk7L17ao
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql /tmp/tmp.oHgClg8g3G/select-4.sql
++ getSecretData internal-some-name operator
++ local secretName=internal-some-name
++ local dataKey=operator
++ kubectl_bin get secrets/internal-some-name '--template={{.data.operator}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FybZCOd0WX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Bym7F6QLEn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/internal-some-name '--template={{.data.operator}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FybZCOd0WX
++ cat /tmp/tmp.Bym7F6QLEn
++ rm /tmp/tmp.FybZCOd0WX /tmp/tmp.Bym7F6QLEn
++ return 0
+ pass=test-password2
+ desc 'check secret without operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
check secret without operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.rMJ6l7JLzy
++ mktemp
+ local LAST_ERR=/tmp/tmp.TLfYwBkAKe
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rMJ6l7JLzy
secret/my-cluster-secrets-2 configured
+ cat /tmp/tmp.TLfYwBkAKe
Warning: resource secrets/my-cluster-secrets-2 is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.rMJ6l7JLzy /tmp/tmp.TLfYwBkAKe
+ return 0
+ sleep 15
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SVtAJC7ona
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TLz7zMDdxr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SVtAJC7ona
++ cat /tmp/tmp.TLz7zMDdxr
++ rm /tmp/tmp.SVtAJC7ona /tmp/tmp.TLz7zMDdxr
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-4.sql /tmp/tmp.oHgClg8g3G/select-4.sql
+ newpass=test-password2
++ echo -n test-password2
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZDI=
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/conf/some-name.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/conf/some-name.yml
++ mktemp
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/conf/some-name.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ local LAST_OUT=/tmp/tmp.0ZQeIyMKgz
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1588-5c1f635c#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-25915~
++ mktemp
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#'
+ local LAST_ERR=/tmp/tmp.WxnHoECsdV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0ZQeIyMKgz
perconaxtradbcluster.pxc.percona.com/some-name configured
+ cat /tmp/tmp.WxnHoECsdV
+ rm /tmp/tmp.0ZQeIyMKgz /tmp/tmp.WxnHoECsdV
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DiqZgES53P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DcbrvpHAtV
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DiqZgES53P
++ cat /tmp/tmp.DcbrvpHAtV
++ rm /tmp/tmp.DiqZgES53P /tmp/tmp.DcbrvpHAtV
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Yy8zuguGQY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RqQHApP78y
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Yy8zuguGQY
++ cat /tmp/tmp.RqQHApP78y
++ rm /tmp/tmp.Yy8zuguGQY /tmp/tmp.RqQHApP78y
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tWG11UN69x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tqp1lnGGqk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tWG11UN69x
++ cat /tmp/tmp.tqp1lnGGqk
++ rm /tmp/tmp.tWG11UN69x /tmp/tmp.tqp1lnGGqk
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.THBzEa4FSB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dR8oxCR1rB
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.THBzEa4FSB
++ cat /tmp/tmp.dR8oxCR1rB
++ rm /tmp/tmp.THBzEa4FSB /tmp/tmp.dR8oxCR1rB
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ocv1KWquJq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.M4dnBq7nUS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ocv1KWquJq
++ cat /tmp/tmp.M4dnBq7nUS
++ rm /tmp/tmp.ocv1KWquJq /tmp/tmp.M4dnBq7nUS
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Sk0xYUYc91
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bJFdU2nr5m
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Sk0xYUYc91
++ cat /tmp/tmp.bJFdU2nr5m
++ rm /tmp/tmp.Sk0xYUYc91 /tmp/tmp.bJFdU2nr5m
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gL7yCmroRm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sihu6adfdz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gL7yCmroRm
++ cat /tmp/tmp.sihu6adfdz
++ rm /tmp/tmp.gL7yCmroRm /tmp/tmp.sihu6adfdz
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XPE1ya1frF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3cx4wJv4rq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XPE1ya1frF
++ cat /tmp/tmp.3cx4wJv4rq
++ rm /tmp/tmp.XPE1ya1frF /tmp/tmp.3cx4wJv4rq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yojyw3VZy7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1vkvC8GheP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.yojyw3VZy7
++ cat /tmp/tmp.1vkvC8GheP
++ rm /tmp/tmp.yojyw3VZy7 /tmp/tmp.1vkvC8GheP
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7QnGrraJrU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0yktt0NVP6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7QnGrraJrU
++ cat /tmp/tmp.0yktt0NVP6
++ rm /tmp/tmp.7QnGrraJrU /tmp/tmp.0yktt0NVP6
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sJiMFBtCVz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PpK6XxNrHU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sJiMFBtCVz
++ cat /tmp/tmp.PpK6XxNrHU
++ rm /tmp/tmp.sJiMFBtCVz /tmp/tmp.PpK6XxNrHU
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.307FiOYDnU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gTq8X40fbq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.307FiOYDnU
++ cat /tmp/tmp.gTq8X40fbq
++ rm /tmp/tmp.307FiOYDnU /tmp/tmp.gTq8X40fbq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LQVvs0pBtx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.36yLJUgjCD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LQVvs0pBtx
++ cat /tmp/tmp.36yLJUgjCD
++ rm /tmp/tmp.LQVvs0pBtx /tmp/tmp.36yLJUgjCD
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.acN2VgCtPR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IHnmpkWuHD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.acN2VgCtPR
++ cat /tmp/tmp.IHnmpkWuHD
++ rm /tmp/tmp.acN2VgCtPR /tmp/tmp.IHnmpkWuHD
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YlttpEzq8C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NYJQnONL8i
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YlttpEzq8C
++ cat /tmp/tmp.NYJQnONL8i
++ rm /tmp/tmp.YlttpEzq8C /tmp/tmp.NYJQnONL8i
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cxSkNvVFQ8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TL3wNwCxrS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cxSkNvVFQ8
++ cat /tmp/tmp.TL3wNwCxrS
++ rm /tmp/tmp.cxSkNvVFQ8 /tmp/tmp.TL3wNwCxrS
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.kUJQ8F4000
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.8vsIUpGkcJ
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.kUJQ8F4000
+++++ cat /tmp/tmp.8vsIUpGkcJ
+++++ rm /tmp/tmp.kUJQ8F4000 /tmp/tmp.8vsIUpGkcJ
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2Z6oYH6a6R
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KbbBIpkjVI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2Z6oYH6a6R
++ cat /tmp/tmp.KbbBIpkjVI
++ rm /tmp/tmp.2Z6oYH6a6R /tmp/tmp.KbbBIpkjVI
++ return 0
+ [[ 3 == \3 ]]
+ patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZDI=
+ local secret=my-cluster-secrets
+ local key=monitor
+ local value=dGVzdC1wYXNzd29yZDI=
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.LNjWfECtAr
++ mktemp
+ local LAST_ERR=/tmp/tmp.lor9NSyvgD
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.LNjWfECtAr
secret/my-cluster-secrets patched
+ cat /tmp/tmp.lor9NSyvgD
+ rm /tmp/tmp.LNjWfECtAr /tmp/tmp.lor9NSyvgD
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cievv4Sfbl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DM9isEVjJM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cievv4Sfbl
++ cat /tmp/tmp.DM9isEVjJM
++ rm /tmp/tmp.cievv4Sfbl /tmp/tmp.DM9isEVjJM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kT3qUrH8bK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zOaKN4loWu
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kT3qUrH8bK
++ cat /tmp/tmp.zOaKN4loWu
++ rm /tmp/tmp.kT3qUrH8bK /tmp/tmp.zOaKN4loWu
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p113uxlDpy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.33mOtrWXCR
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.p113uxlDpy
++ cat /tmp/tmp.33mOtrWXCR
++ rm /tmp/tmp.p113uxlDpy /tmp/tmp.33mOtrWXCR
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.igPMmvBUKx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zCJVK3SaIc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.igPMmvBUKx
++ cat /tmp/tmp.zCJVK3SaIc
++ rm /tmp/tmp.igPMmvBUKx /tmp/tmp.zCJVK3SaIc
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SrKOgejjPR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ORd3KnYZnp
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SrKOgejjPR
++ cat /tmp/tmp.ORd3KnYZnp
++ rm /tmp/tmp.SrKOgejjPR /tmp/tmp.ORd3KnYZnp
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.fIkGqsIKhg
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.fQQEN8HDZY
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.fIkGqsIKhg
+++++ cat /tmp/tmp.fQQEN8HDZY
+++++ rm /tmp/tmp.fIkGqsIKhg /tmp/tmp.fQQEN8HDZY
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lvNLhdsnGl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FbpE2hcI1u
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.lvNLhdsnGl
++ cat /tmp/tmp.FbpE2hcI1u
++ rm /tmp/tmp.lvNLhdsnGl /tmp/tmp.FbpE2hcI1u
++ return 0
+ [[ 3 == \3 ]]
+ compare_mysql_cmd select-3 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local command_id=select-3
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ run_mysql 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6WR5Z8OoG1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s6YOQ5yPcO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6WR5Z8OoG1
++ cat /tmp/tmp.s6YOQ5yPcO
++ rm /tmp/tmp.6WR5Z8OoG1 /tmp/tmp.s6YOQ5yPcO
++ return 0
+ client_pod=pxc-client-7b8b9b895-dj8t4
+ wait_pod pxc-client-7b8b9b895-dj8t4
+ local pod=pxc-client-7b8b9b895-dj8t4
+ local max_retry=480
+ local ns=
++ echo pxc-client-7b8b9b895-dj8t4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-7b8b9b895-dj8t4 condition met
pxc-client-7b8b9b895-dj8t4.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.oHgClg8g3G/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1588/e2e-tests/users/compare/select-3.sql /tmp/tmp.oHgClg8g3G/select-3.sql
+ destroy users-25915
+ local namespace=users-25915
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ grep -v level=info
++ get_operator_pod
+ grep -v 'get backup status: Job.batch'
++ local label_prefix=app.kubernetes.io/
+ sort -u
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
+ tee /tmp/tmp.oHgClg8g3G/operator.log
+ grep -v 'the object has been modified'
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Tp4phQzz8A
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7LhlwcRM5h
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Tp4phQzz8A
++ cat /tmp/tmp.7LhlwcRM5h
++ rm /tmp/tmp.Tp4phQzz8A /tmp/tmp.7LhlwcRM5h
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-66688f478-8czhs
++ mktemp
+ local LAST_OUT=/tmp/tmp.BVzlencuBR
++ mktemp
+ local LAST_ERR=/tmp/tmp.SXGpl0hU7q
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-66688f478-8czhs
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.BVzlencuBR
+ cat /tmp/tmp.SXGpl0hU7q
+ rm /tmp/tmp.BVzlencuBR /tmp/tmp.SXGpl0hU7q
+ return 0
2024-02-05T15:15:54.329Z	INFO	setup	Manager starting up	{"gitCommit": "5c1f635cea7dcb3dc6d1b248cb6465a7095a4f8e", "gitBranch": "PR-1588-5c1f635c", "buildTime": "2024-02-05T12:49:15Z", "goVersion": "go1.21.6", "os": "linux", "arch": "amd64"}
2024-02-05T15:15:54.329Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.25.16-gke.1360000"}
2024-02-05T15:15:54.330Z	INFO	setup	Registering Components.
2024-02-05T15:15:57.014Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2024-02-05T15:15:57.014Z	INFO	controller-runtime.metrics	Starting metrics server
2024-02-05T15:15:57.014Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2024-02-05T15:15:57.014Z	INFO	controller-runtime.webhook	Starting webhook server
2024-02-05T15:15:57.014Z	INFO	setup	Starting the Cmd.
2024-02-05T15:15:57.014Z	INFO	starting server	{"kind": "health probe", "addr": "[::]:8081"}
2024-02-05T15:15:57.015Z	INFO	attempting to acquire leader lease pxc-operator/08db0feb.percona.com...
2024-02-05T15:15:57.015Z	INFO	controller-runtime.certwatcher	Starting certificate watcher
2024-02-05T15:15:57.015Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate
2024-02-05T15:15:57.015Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2024-02-05T15:15:57.043Z	DEBUG	events	percona-xtradb-cluster-operator-66688f478-8czhs_1648937c-abb7-4eb6-a9bf-4fcfc0c62ee4 became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db0feb.percona.com","uid":"22607297-3a31-4097-8f25-d85c389a1a6e","apiVersion":"coordination.k8s.io/v1","resourceVersion":"76115"}, "reason": "LeaderElection"}
2024-02-05T15:15:57.043Z	INFO	successfully acquired lease pxc-operator/08db0feb.percona.com
2024-02-05T15:15:57.044Z	INFO	Starting Controller	{"controller": "pxcbackup-controller"}
2024-02-05T15:15:57.044Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2024-02-05T15:15:57.044Z	INFO	Starting Controller	{"controller": "pxcrestore-controller"}
2024-02-05T15:15:57.044Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2024-02-05T15:15:57.044Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2024-02-05T15:15:57.044Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2024-02-05T15:15:57.260Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "worker count": 1}
2024-02-05T15:15:57.260Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2024-02-05T15:15:57.261Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "worker count": 1}
2024-02-05T15:16:28.157Z	INFO	Set CR version	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "341af1fa-a005-45f2-9952-17a7017ae821", "version": "1.14.0"}
2024-02-05T15:17:47.132Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "1f400533-dae5-4ba0-a6a7-0968c0f5f396", "user": "operator"}
2024-02-05T15:17:47.169Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "1f400533-dae5-4ba0-a6a7-0968c0f5f396", "user": "monitor"}
2024-02-05T15:17:47.207Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "1f400533-dae5-4ba0-a6a7-0968c0f5f396"}
2024-02-05T15:17:47.248Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "1f400533-dae5-4ba0-a6a7-0968c0f5f396", "user": "xtrabackup"}
2024-02-05T15:17:47.280Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "1f400533-dae5-4ba0-a6a7-0968c0f5f396"}
2024-02-05T15:17:47.361Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "1f400533-dae5-4ba0-a6a7-0968c0f5f396", "err": "get primary pxc pod: not found"}
2024-02-05T15:17:52.206Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "bfe52854-6cdf-48e0-86d7-9e369a5bffbb", "err": "get primary pxc pod: not found"}
2024-02-05T15:17:57.384Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "88625faf-696d-4bae-bec2-8885fd53420d", "err": "get primary pxc pod: not found"}
2024-02-05T15:20:14.655Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "01046762-633f-4eba-9d2d-3ea0db366372", "user": "root"}
2024-02-05T15:20:14.706Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "01046762-633f-4eba-9d2d-3ea0db366372", "user": "replication"}
2024-02-05T15:20:14.914Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "01046762-633f-4eba-9d2d-3ea0db366372", "new version": "5.7.44-48-57"}
2024-02-05T15:20:16.541Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "01046762-633f-4eba-9d2d-3ea0db366372"}
2024-02-05T15:20:21.429Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "747ba74c-234d-4764-af26-46c017d0784c"}
2024-02-05T15:20:26.698Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "0c578f70-c4df-40d3-bf7d-4c7aa750c760"}
2024-02-05T15:20:31.956Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "1e14262c-b68c-43e1-aa24-9ffc794333fe"}
2024-02-05T15:20:37.226Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "94e13fea-0ec7-4861-8e93-b61cb7628899"}
2024-02-05T15:20:42.540Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "dedead5d-c2b5-4ad8-bacf-31d13cd820d7"}
2024-02-05T15:20:47.744Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "4b33f563-e855-4679-b424-b7eb577e8c8e"}
2024-02-05T15:20:53.035Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "8616df52-d350-4740-9c0d-4bfab530001c"}
2024-02-05T15:20:58.248Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "5f916a94-e3ca-4899-a72f-c30d82987c64"}
2024-02-05T15:21:03.629Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "0d1482d8-4743-42b8-8e2c-3485f660cea0"}
2024-02-05T15:21:08.855Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "b474de98-b9fd-4dae-9c67-5b805d1a3045"}
2024-02-05T15:21:14.163Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "22586180-8421-4df2-a419-d095cecb8d78"}
2024-02-05T15:21:17.532Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "9d1bc49b-5810-4d9a-8958-75e821861ece", "user": "root"}
2024-02-05T15:21:17.570Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "9d1bc49b-5810-4d9a-8958-75e821861ece", "user": "root"}
2024-02-05T15:21:17.577Z	INFO	MySQL init secret created	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "9d1bc49b-5810-4d9a-8958-75e821861ece", "secret": "some-name-mysql-init", "user": "root"}
2024-02-05T15:21:20.590Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "9d1bc49b-5810-4d9a-8958-75e821861ece"}
2024-02-05T15:21:20.599Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "9d1bc49b-5810-4d9a-8958-75e821861ece", "user": "root"}
2024-02-05T15:21:22.940Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "9d1bc49b-5810-4d9a-8958-75e821861ece"}
2024-02-05T15:21:27.869Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "6202df2a-96b1-4339-99a9-0f9b4c89883e"}
2024-02-05T15:21:32.952Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "31960ea9-06ac-4c6f-858d-19b97705bb71"}
2024-02-05T15:21:54.756Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "3fea4bed-78e7-4049-9a8f-310dcb16b13b", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:387) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR (line:387) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:21:57.215Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "7bb338b6-871b-4a7c-8a77-d04a650c87b9", "user": "proxyadmin"}
2024-02-05T15:21:57.215Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "7bb338b6-871b-4a7c-8a77-d04a650c87b9", "user": "proxyadmin"}
2024-02-05T15:21:57.288Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "7bb338b6-871b-4a7c-8a77-d04a650c87b9", "user": "proxyadmin"}
2024-02-05T15:21:57.298Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "7bb338b6-871b-4a7c-8a77-d04a650c87b9", "user": "proxyadmin"}
2024-02-05T15:21:57.298Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "7bb338b6-871b-4a7c-8a77-d04a650c87b9", "last-applied-secret": "1a55867d0f4fa0d0643653b2eb57b2f690315d8d843a10599c9839c2a779a95f"}
2024-02-05T15:21:57.465Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "7bb338b6-871b-4a7c-8a77-d04a650c87b9", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:22:13.322Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "404f58bb-2d40-4116-a768-29c1a7a4c81c", "err": "get primary pxc pod: not found"}
2024-02-05T15:22:18.518Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "c2bdfdbb-79f3-4f0f-b470-d84349da5a7d", "err": "get primary pxc pod: not found"}
2024-02-05T15:22:50.283Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "698a3274-4d26-4668-804f-fee5a6caf736"}
2024-02-05T15:22:55.263Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "e234875f-c9fd-4d49-baf1-3344feaa621f"}
2024-02-05T15:23:00.383Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "88e89104-b3df-4a2b-899d-314195abf708"}
2024-02-05T15:23:05.797Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "769513e5-ff4b-404a-94e7-82a115c3cd30"}
2024-02-05T15:23:08.474Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d6265013-883a-44e5-b6ad-0d6a9e1b082e"}
2024-02-05T15:23:13.795Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "cb8f0c95-948d-427b-8cca-a5017517dc9a", "user": "xtrabackup"}
2024-02-05T15:23:13.820Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "cb8f0c95-948d-427b-8cca-a5017517dc9a", "user": "xtrabackup"}
2024-02-05T15:23:13.832Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "cb8f0c95-948d-427b-8cca-a5017517dc9a", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-02-05T15:23:13.841Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "cb8f0c95-948d-427b-8cca-a5017517dc9a", "user": "xtrabackup"}
2024-02-05T15:23:13.841Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "cb8f0c95-948d-427b-8cca-a5017517dc9a", "last-applied-secret": "d11bcc70ca53ea410cd455e13d82e7225917b0861d9b677ed34c02c5f6fbaf1b"}
2024-02-05T15:23:16.485Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "cb8f0c95-948d-427b-8cca-a5017517dc9a"}
2024-02-05T15:25:18.445Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "e0eb999d-86af-4b96-abbc-84ca536dbc67", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-25915 on 10.41.64.10:53: no such host"}
2024-02-05T15:25:23.731Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "09bff339-7b14-4b6e-8e41-c592e10480dc", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-25915 on 10.41.64.10:53: no such host"}
2024-02-05T15:25:28.910Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f985a6fd-0881-48cd-b530-72370cebb911", "primary name": "some-name-pxc-0.some-name-pxc.users-25915.svc.cluster.local"}
2024-02-05T15:25:34.050Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "b97ff898-2323-4576-8beb-4954a9ebe10b", "primary name": "some-name-pxc-0.some-name-pxc.users-25915.svc.cluster.local"}
2024-02-05T15:25:39.255Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "c82f1722-4d44-4521-8518-e8263a573312", "primary name": "some-name-pxc-0.some-name-pxc.users-25915.svc.cluster.local"}
2024-02-05T15:25:44.470Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "1a71e9d9-f22a-48d0-9e1d-a10bff55f94f", "primary name": "some-name-pxc-0.some-name-pxc.users-25915.svc.cluster.local"}
2024-02-05T15:25:49.614Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "ce0de6d9-fc65-4e29-918d-c6fdc8d39131", "primary name": "some-name-pxc-0.some-name-pxc.users-25915.svc.cluster.local"}
2024-02-05T15:25:54.772Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "27eb2fc6-d667-46c3-9623-e17fd95eb2ce", "primary name": "some-name-pxc-0.some-name-pxc.users-25915.svc.cluster.local"}
2024-02-05T15:25:59.922Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f407dfa2-f489-4f6d-b8ee-f5df0c985774", "primary name": "some-name-pxc-0.some-name-pxc.users-25915.svc.cluster.local"}
2024-02-05T15:26:07.134Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "1f0d3732-75b5-4cd1-94cd-71480a818b6e"}
2024-02-05T15:26:12.134Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "cc6dce48-daf4-4701-948b-538ce36aa565"}
2024-02-05T15:26:17.270Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "1edd3baf-e698-45c6-84c1-3f83685618c8"}
2024-02-05T15:26:20.724Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "4ffe0d81-96b6-4cae-a115-abea674214fd", "user": "monitor"}
2024-02-05T15:26:20.752Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "4ffe0d81-96b6-4cae-a115-abea674214fd", "user": "monitor"}
2024-02-05T15:26:20.760Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "4ffe0d81-96b6-4cae-a115-abea674214fd", "secret": "some-name-mysql-init", "user": "monitor"}
2024-02-05T15:26:20.808Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "4ffe0d81-96b6-4cae-a115-abea674214fd", "user": "monitor"}
2024-02-05T15:26:20.815Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "4ffe0d81-96b6-4cae-a115-abea674214fd", "user": "monitor"}
2024-02-05T15:26:20.815Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "4ffe0d81-96b6-4cae-a115-abea674214fd", "last-applied-secret": "4e3282050745046d5044e0fcb5f2bf140d7320a1256a79c8ca2184f38f4f3098"}
2024-02-05T15:26:22.622Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "4ffe0d81-96b6-4cae-a115-abea674214fd", "error": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / ", "errorVerbose": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:27:02.984Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "9a95031b-f9a2-4e6a-9687-d2e001c2908c"}
2024-02-05T15:27:07.998Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "b38ff6fa-b826-4bc6-bfd1-0cefcc39ff66"}
2024-02-05T15:27:13.359Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "3912617b-1abf-44d1-b11c-568f36e6c2a2"}
2024-02-05T15:27:26.003Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "6794f831-ec9d-47b9-af37-f26aa7991883"}
2024-02-05T15:27:29.305Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "32db2ed0-674b-436b-bcfb-85fc9ecb72c3", "user": "operator"}
2024-02-05T15:27:29.332Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "32db2ed0-674b-436b-bcfb-85fc9ecb72c3", "user": "operator"}
2024-02-05T15:27:29.341Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "32db2ed0-674b-436b-bcfb-85fc9ecb72c3", "secret": "some-name-mysql-init", "user": "operator"}
2024-02-05T15:27:29.368Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "32db2ed0-674b-436b-bcfb-85fc9ecb72c3", "user": "operator"}
2024-02-05T15:27:29.368Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "32db2ed0-674b-436b-bcfb-85fc9ecb72c3", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-02-05T15:27:30.047Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "32db2ed0-674b-436b-bcfb-85fc9ecb72c3", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-25915.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-25915.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-25915.svc.' (using password: YES)\nERROR (line:1437) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-25915.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-25915.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-25915.svc.' (using password: YES)\nERROR (line:1437) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:28:09.871Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "e3a24152-3f57-4bdf-a53c-7ad9277a7c3e"}
2024-02-05T15:28:13.950Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "9b42e775-3636-4f76-a4de-0dfdd7dd7e54"}
2024-02-05T15:28:19.157Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "98706fae-73c1-4eae-9a77-d29d88d7f0e8"}
2024-02-05T15:28:24.458Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "364647e2-c11b-47d6-8a8c-fc69068c9acb"}
2024-02-05T15:28:24.543Z	INFO	Created user secrets	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "secrets": "my-cluster-secrets-2"}
2024-02-05T15:28:24.543Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "root"}
2024-02-05T15:28:24.578Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "root"}
2024-02-05T15:28:24.586Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "secret": "some-name-mysql-init", "user": "root"}
2024-02-05T15:28:27.415Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9"}
2024-02-05T15:28:27.425Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "root"}
2024-02-05T15:28:27.425Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "operator"}
2024-02-05T15:28:27.450Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "operator"}
2024-02-05T15:28:27.459Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "secret": "some-name-mysql-init", "user": "operator"}
2024-02-05T15:28:27.470Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "operator"}
2024-02-05T15:28:27.470Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "monitor"}
2024-02-05T15:28:27.494Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "monitor"}
2024-02-05T15:28:27.501Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "secret": "some-name-mysql-init", "user": "monitor"}
2024-02-05T15:28:27.545Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "monitor"}
2024-02-05T15:28:27.554Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "monitor"}
2024-02-05T15:28:27.554Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "xtrabackup"}
2024-02-05T15:28:27.579Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "xtrabackup"}
2024-02-05T15:28:27.585Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-02-05T15:28:27.594Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "xtrabackup"}
2024-02-05T15:28:27.594Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "replication"}
2024-02-05T15:28:27.674Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "replication"}
2024-02-05T15:28:27.681Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "secret": "some-name-mysql-init", "user": "replication"}
2024-02-05T15:28:27.689Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "replication"}
2024-02-05T15:28:27.689Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "proxyadmin"}
2024-02-05T15:28:27.735Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "proxyadmin"}
2024-02-05T15:28:27.745Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "user": "proxyadmin"}
2024-02-05T15:28:27.745Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "last-applied-secret": "88e1f771592abf24f525a9667a69fa7b44868ef3e0fa0fd42cc5a38c3d6fa657"}
2024-02-05T15:28:27.745Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "last-applied-secret": "88e1f771592abf24f525a9667a69fa7b44868ef3e0fa0fd42cc5a38c3d6fa657"}
2024-02-05T15:28:27.966Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f88d506f-6889-4a62-9221-dc53762973b9", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:30:22.943Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "9bb4495b-a0fc-4cbc-bebb-33070fb620aa", "err": "failed to connect to pod some-name-pxc-0: dial tcp 10.41.49.16:33062: connect: connection refused"}
2024-02-05T15:30:28.077Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "4b866429-ac1a-430c-81a6-c3b5d3fc7d5d", "primary name": "some-name-pxc-0.some-name-pxc.users-25915.svc.cluster.local"}
2024-02-05T15:31:11.877Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "cb0b7dc5-64fd-4f2d-a28c-8a55752e40a6"}
2024-02-05T15:31:16.857Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "e7e99907-6be2-45da-aefb-8e3e97aa9d56"}
2024-02-05T15:31:22.083Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "fb3e9286-9330-41e2-867e-d3b0273336ec"}
2024-02-05T15:31:27.358Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "28d02e49-21dc-42d5-bc33-d696e1fa604a"}
2024-02-05T15:31:32.588Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "49ec7b54-8676-4c99-9490-7856102eccbb"}
2024-02-05T15:31:35.900Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d49f6c74-16e6-47e2-ab0f-d8b1b200ab18", "user": "operator"}
2024-02-05T15:31:35.930Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d49f6c74-16e6-47e2-ab0f-d8b1b200ab18", "user": "operator"}
2024-02-05T15:31:35.936Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d49f6c74-16e6-47e2-ab0f-d8b1b200ab18", "secret": "some-name-mysql-init", "user": "operator"}
2024-02-05T15:31:35.945Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d49f6c74-16e6-47e2-ab0f-d8b1b200ab18", "user": "operator"}
2024-02-05T15:31:35.945Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d49f6c74-16e6-47e2-ab0f-d8b1b200ab18", "last-applied-secret": "20933199e1341f6cb29c601ba1bc1b4032aa18ceab2b03805a005f820caf1404"}
2024-02-05T15:31:36.578Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d49f6c74-16e6-47e2-ab0f-d8b1b200ab18", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-25915.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-25915.svc.' (using password: YES)\nERROR (line:1437) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-25915.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-25915.svc.' (using password: YES)\nERROR (line:1437) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:32:21.483Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "533ba716-955a-4a26-a655-9adff3ffa29f"}
2024-02-05T15:32:25.653Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "29a36161-4ddd-430a-8f33-957c1f472fcf"}
2024-02-05T15:32:30.868Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "c55e4f9d-92b2-4262-bae7-5313d49cc06d"}
2024-02-05T15:32:36.263Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "c1fcf5fe-71fc-42d9-af45-88a65dd55eb6"}
2024-02-05T15:32:41.284Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "0162330a-43e8-45a5-a798-64d231c45538"}
2024-02-05T15:32:46.506Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "71ea7516-bb4d-4d36-a84b-8a99a9a175fd"}
2024-02-05T15:32:51.895Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "8b2c42f0-33fa-401c-ad5e-a9e2edd03753"}
2024-02-05T15:32:57.096Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "c0adb85f-d016-4136-90a0-28e0a7a64a20"}
2024-02-05T15:33:02.351Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "8d997685-d5ac-4747-8118-f0da9881a440"}
2024-02-05T15:33:07.502Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "25bc7294-677b-474f-b8b6-f67dcfc2da8f"}
2024-02-05T15:33:13.638Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "f82540ce-bf50-489d-bb9f-d7dcadd2e518"}
2024-02-05T15:33:18.052Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "e452bcec-5458-4bf1-8ae6-e40fe6f8e353"}
2024-02-05T15:33:23.279Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d5b317fd-df52-45e8-bdd6-0ac458e98493"}
2024-02-05T15:33:28.891Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "a194b5e7-df5f-4798-a5a1-757d21fb6189"}
2024-02-05T15:33:34.002Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "3123d615-0354-41eb-a20b-d09c2c707ccd"}
2024-02-05T15:33:39.457Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "4b7f5fa7-bfba-4ce6-a86c-f6e69921fbd1"}
2024-02-05T15:33:44.601Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "6d4fe06b-2b1e-4b8b-82e7-1dca7aeb8b7b"}
2024-02-05T15:33:49.875Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "29010307-5976-44d1-8e54-c030c2f2025e"}
2024-02-05T15:33:55.007Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "0890b4c2-cf04-4f37-b3d9-a5223a2fc965"}
2024-02-05T15:33:58.570Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "root"}
2024-02-05T15:33:58.602Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "root"}
2024-02-05T15:33:58.608Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "secret": "some-name-mysql-init", "user": "root"}
2024-02-05T15:34:01.582Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0"}
2024-02-05T15:34:01.590Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "root"}
2024-02-05T15:34:01.591Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "monitor"}
2024-02-05T15:34:01.616Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "monitor"}
2024-02-05T15:34:01.622Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "secret": "some-name-mysql-init", "user": "monitor"}
2024-02-05T15:34:01.665Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "monitor"}
2024-02-05T15:34:01.673Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "monitor"}
2024-02-05T15:34:01.673Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "xtrabackup"}
2024-02-05T15:34:01.702Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "xtrabackup"}
2024-02-05T15:34:01.709Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-02-05T15:34:01.716Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "xtrabackup"}
2024-02-05T15:34:01.716Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "proxyadmin"}
2024-02-05T15:34:01.763Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "proxyadmin"}
2024-02-05T15:34:01.775Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "user": "proxyadmin"}
2024-02-05T15:34:01.775Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "last-applied-secret": "dc3ee4fd1fc94e87c6bfcc1f78622bd6173e416b11c807e7bf9669c23f490a21"}
2024-02-05T15:34:01.775Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "last-applied-secret": "dc3ee4fd1fc94e87c6bfcc1f78622bd6173e416b11c807e7bf9669c23f490a21"}
2024-02-05T15:34:01.993Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "035e8f7d-6545-49ff-b079-2ba7631954c0", "error": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1233\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"}
2024-02-05T15:35:41.249Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "bafae2ba-0e90-4d2e-99d5-6628f747bb70", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-25915 on 10.41.64.10:53: no such host"}
2024-02-05T15:35:46.290Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "bb345aff-53b3-4ffa-b457-4d2d8b70a382", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-25915 on 10.41.64.10:53: no such host"}
2024-02-05T15:35:51.498Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "20d3b009-cb05-42ff-b016-ac4177f1e148", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-25915 on 10.41.64.10:53: no such host"}
2024-02-05T15:35:51.723Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "ddc547b4-2626-4503-bf5f-cf40482fcd28", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-25915 on 10.41.64.10:53: no such host"}
2024-02-05T15:35:56.846Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "2accd5a0-121a-4d61-bb63-30b07a516cfc", "primary name": "some-name-pxc-0.some-name-pxc.users-25915.svc.cluster.local"}
2024-02-05T15:36:02.138Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d7291240-7d00-47b1-a69a-b8f863191a39", "primary name": "some-name-pxc-0.some-name-pxc.users-25915.svc.cluster.local"}
2024-02-05T15:36:07.333Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "0a436db7-9781-4952-9454-cfcc712aa440", "primary name": "some-name-pxc-0.some-name-pxc.users-25915.svc.cluster.local"}
2024-02-05T15:36:12.500Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "431b60cf-0477-4f73-abe1-18f85102ad62", "primary name": "some-name-pxc-0.some-name-pxc.users-25915.svc.cluster.local"}
2024-02-05T15:36:24.972Z	INFO	KubeAPIWarningLogger	.metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: ede5aea9-aed2-4aab-8b98-b82e4411cbb6
2024-02-05T15:36:28.160Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "1ef69965-d4f8-4d80-8675-89eb8d219926", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.41.71.85:3306: connect: connection refused"}
2024-02-05T15:38:13.265Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "e2877ee5-7245-43f3-8fbb-20e609ca0325", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-25915 on 10.41.64.10:53: no such host"}
2024-02-05T15:38:13.674Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "3eb94607-749f-49a2-a2e1-a43eaa52ac65", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-25915 on 10.41.64.10:53: no such host"}
2024-02-05T15:38:18.573Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "70ed62b7-416f-4c56-a07c-ce5c8754ba26", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-25915 on 10.41.64.10:53: no such host"}
2024-02-05T15:39:10.391Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "root"}
2024-02-05T15:39:10.425Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "root"}
2024-02-05T15:39:10.434Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "secret": "some-name-mysql-init", "user": "root"}
2024-02-05T15:39:10.442Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "root"}
2024-02-05T15:39:10.442Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "operator"}
2024-02-05T15:39:10.464Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "operator"}
2024-02-05T15:39:10.474Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "secret": "some-name-mysql-init", "user": "operator"}
2024-02-05T15:39:10.482Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "operator"}
2024-02-05T15:39:10.482Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "monitor"}
2024-02-05T15:39:10.504Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "monitor"}
2024-02-05T15:39:10.512Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "secret": "some-name-mysql-init", "user": "monitor"}
2024-02-05T15:39:10.526Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "monitor"}
2024-02-05T15:39:10.526Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "xtrabackup"}
2024-02-05T15:39:10.548Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "xtrabackup"}
2024-02-05T15:39:10.556Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-02-05T15:39:10.567Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "xtrabackup"}
2024-02-05T15:39:10.567Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "replication"}
2024-02-05T15:39:10.594Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "replication"}
2024-02-05T15:39:10.601Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "secret": "some-name-mysql-init", "user": "replication"}
2024-02-05T15:39:10.610Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "user": "replication"}
2024-02-05T15:39:10.610Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-02-05T15:39:10.611Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "46aa6ae0-c2bd-4ceb-af17-10082d7b0c65", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-02-05T15:41:46.802Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d108a1f0-189b-4bd5-811e-6e3c791528b6", "user": "monitor"}
2024-02-05T15:41:46.829Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d108a1f0-189b-4bd5-811e-6e3c791528b6", "user": "monitor"}
2024-02-05T15:41:46.835Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d108a1f0-189b-4bd5-811e-6e3c791528b6", "secret": "some-name-mysql-init", "user": "monitor"}
2024-02-05T15:41:46.844Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d108a1f0-189b-4bd5-811e-6e3c791528b6", "user": "monitor"}
2024-02-05T15:41:46.844Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-25915", "name": "some-name", "reconcileID": "d108a1f0-189b-4bd5-811e-6e3c791528b6", "last-applied-secret": "81935ecf30ae523a6ee69582354f9488d47dbeb0f8c7c572af96870f2024284d"}
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:227
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:266
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:329
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1235
[mysql] 2024/02/05 15:26:21 packets.go:37: unexpected EOF
[mysql] 2024/02/05 15:27:29 packets.go:37: unexpected EOF
[mysql] 2024/02/05 15:38:38 packets.go:37: read tcp 10.41.48.54:50712->10.41.71.85:3306: i/o timeout
[mysql] 2024/02/05 15:41:10 packets.go:37: read tcp 10.41.48.54:35182->10.41.71.85:3306: i/o timeout
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n users-25915 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.PW6IuUhm7L
++ mktemp
+ local LAST_ERR=/tmp/tmp.KIJIL1utmJ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.PW6IuUhm7L
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.KIJIL1utmJ
+ rm /tmp/tmp.PW6IuUhm7L /tmp/tmp.KIJIL1utmJ
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.c0D3zNwTo0
++ mktemp
+ local LAST_ERR=/tmp/tmp.6tKk6WUNZ4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.c0D3zNwTo0
No resources found
+ cat /tmp/tmp.6tKk6WUNZ4
+ rm /tmp/tmp.c0D3zNwTo0 /tmp/tmp.6tKk6WUNZ4
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.hITI7XlZ9r
++ mktemp
+ local LAST_ERR=/tmp/tmp.F91bzjBAJ1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.hITI7XlZ9r
No resources found
+ cat /tmp/tmp.F91bzjBAJ1
+ rm /tmp/tmp.hITI7XlZ9r /tmp/tmp.F91bzjBAJ1
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.m4epinWKEt
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZahUsq2xbo
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.m4epinWKEt
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.ZahUsq2xbo
+ rm /tmp/tmp.m4epinWKEt /tmp/tmp.ZahUsq2xbo
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.12.1/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace users-25915
+ rm -rf /tmp/tmp.oHgClg8g3G
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.LsbYyRrg0F
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.TilDCkVN2P
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.SvyMMOGOFt
+ local exit_status=0
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.tDI1f8qrRc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace users-25915
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator