Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/logs/users-5-7.log
WARNING: version difference between client (1.32) and server (1.28) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.32) and server (1.28) exceeds the supported minor version skew of +/-1
+ create_infra users-7894
+ local ns=users-7894
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n users-3218 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.ygHsPc1Mi4
++ mktemp
+ local LAST_ERR=/tmp/tmp.qpMKikWXba
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ygHsPc1Mi4
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.qpMKikWXba
+ rm /tmp/tmp.ygHsPc1Mi4 /tmp/tmp.qpMKikWXba
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.xmR0l1PqYL
++ mktemp
+ local LAST_ERR=/tmp/tmp.dxXL1VUiN9
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xmR0l1PqYL
No resources found
+ cat /tmp/tmp.dxXL1VUiN9
+ rm /tmp/tmp.xmR0l1PqYL /tmp/tmp.dxXL1VUiN9
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.06A1sKslhz
++ mktemp
+ local LAST_ERR=/tmp/tmp.4XVD3DKnLf
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.06A1sKslhz
No resources found
+ cat /tmp/tmp.4XVD3DKnLf
+ rm /tmp/tmp.06A1sKslhz /tmp/tmp.4XVD3DKnLf
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.VM1ZQuMHHO
++ mktemp
+ local LAST_OUT=/tmp/tmp.BVAy9wgDmp
+ local LAST_ERR=/tmp/tmp.HN9KmevUnE
+ local exit_status=0
++ mktemp
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.Ehqjg5up5N
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.VM1ZQuMHHO
+ cat /tmp/tmp.HN9KmevUnE
+ rm /tmp/tmp.VM1ZQuMHHO /tmp/tmp.HN9KmevUnE
+ return 0
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
namespace "users-3218" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.BVAy9wgDmp
namespace "pxc-operator" deleted
+ cat /tmp/tmp.Ehqjg5up5N
+ rm /tmp/tmp.BVAy9wgDmp /tmp/tmp.Ehqjg5up5N
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.0IHlmouoyv
++ mktemp
+ local LAST_ERR=/tmp/tmp.xaANcZ1FRY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0IHlmouoyv
namespace/pxc-operator created
+ cat /tmp/tmp.xaANcZ1FRY
+ rm /tmp/tmp.0IHlmouoyv /tmp/tmp.xaANcZ1FRY
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LsWWFCcWlo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vzfi8ZvboH
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LsWWFCcWlo
++ cat /tmp/tmp.Vzfi8ZvboH
++ rm /tmp/tmp.LsWWFCcWlo /tmp/tmp.Vzfi8ZvboH
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster9 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.QeBAv4AWve
++ mktemp
+ local LAST_ERR=/tmp/tmp.wkHpVgODY2
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster9 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QeBAv4AWve
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster9" modified.
+ cat /tmp/tmp.wkHpVgODY2
+ rm /tmp/tmp.QeBAv4AWve /tmp/tmp.wkHpVgODY2
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZfW201YywX
++ mktemp
+ local LAST_ERR=/tmp/tmp.D8VdI96h49
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ZfW201YywX
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.D8VdI96h49
+ rm /tmp/tmp.ZfW201YywX /tmp/tmp.D8VdI96h49
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/deploy/cw-rbac.yaml
+ kubectl_bin apply -f -
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
++ mktemp
+ local LAST_OUT=/tmp/tmp.s3ZFfL1XCV
++ mktemp
+ local LAST_ERR=/tmp/tmp.VBBs7hq8qd
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.s3ZFfL1XCV
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.VBBs7hq8qd
+ rm /tmp/tmp.s3ZFfL1XCV /tmp/tmp.VBBs7hq8qd
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1697-26afa527^'
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ewd3EPAtNI
++ mktemp
+ local LAST_ERR=/tmp/tmp.ExzE4PO2gF
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Ewd3EPAtNI
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.ExzE4PO2gF
+ rm /tmp/tmp.Ewd3EPAtNI /tmp/tmp.ExzE4PO2gF
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.WbuEL32zIN
++ mktemp
+ local LAST_ERR=/tmp/tmp.Bs5Xfot7oO
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WbuEL32zIN
pod/percona-xtradb-cluster-operator-67cbddd7f9-wzd8m condition met
+ cat /tmp/tmp.Bs5Xfot7oO
+ rm /tmp/tmp.WbuEL32zIN /tmp/tmp.Bs5Xfot7oO
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bVo3uF4MkY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QiVKPhcIE1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bVo3uF4MkY
++ cat /tmp/tmp.QiVKPhcIE1
++ rm /tmp/tmp.bVo3uF4MkY /tmp/tmp.QiVKPhcIE1
++ return 0
+ wait_pod percona-xtradb-cluster-operator-67cbddd7f9-wzd8m 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-67cbddd7f9-wzd8m
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-67cbddd7f9-wzd8m
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-67cbddd7f9-wzd8m condition met
waiting for pod/percona-xtradb-cluster-operator-67cbddd7f9-wzd8m to become Ready.Ok
+ sleep 3
+ create_namespace users-7894
+ local namespace=users-7894
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces users-7894'
++ mktemp
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces users-7894
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace users-7894
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.cUbunGzIvM
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.L2Hd4YizgF
++ mktemp
+ local LAST_ERR=/tmp/tmp.fTy2hVqelj
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.cUNBGNN633
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-7894
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.cUbunGzIvM
+ cat /tmp/tmp.fTy2hVqelj
+ rm /tmp/tmp.cUbunGzIvM /tmp/tmp.fTy2hVqelj
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-7894
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace users-7894
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.L2Hd4YizgF
+ cat /tmp/tmp.cUNBGNN633
Error from server (NotFound): namespaces "users-7894" not found
+ rm /tmp/tmp.L2Hd4YizgF /tmp/tmp.cUNBGNN633
+ return 1
+ :
+ wait_for_delete namespace/users-7894
+ local res=namespace/users-7894
+ echo -n 'waiting for namespace/users-7894 to be deleted'
waiting for namespace/users-7894 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "users-7894" not found
+ desc 'create namespace users-7894'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace users-7894
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace users-7894
++ mktemp
+ local LAST_OUT=/tmp/tmp.EVJGsdnhbI
++ mktemp
+ local LAST_ERR=/tmp/tmp.xLHuIJzr90
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace users-7894
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EVJGsdnhbI
namespace/users-7894 created
+ cat /tmp/tmp.xLHuIJzr90
+ rm /tmp/tmp.EVJGsdnhbI /tmp/tmp.xLHuIJzr90
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E2WwwrKYD2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kq7RIEDhUL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.E2WwwrKYD2
++ cat /tmp/tmp.kq7RIEDhUL
++ rm /tmp/tmp.E2WwwrKYD2 /tmp/tmp.kq7RIEDhUL
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster9 --namespace=users-7894
++ mktemp
+ local LAST_OUT=/tmp/tmp.xcsVQQJzi5
++ mktemp
+ local LAST_ERR=/tmp/tmp.GrxJDiJDuQ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster9 --namespace=users-7894
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xcsVQQJzi5
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster9" modified.
+ cat /tmp/tmp.GrxJDiJDuQ
+ rm /tmp/tmp.xcsVQQJzi5 /tmp/tmp.GrxJDiJDuQ
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.odcyRAPA8g
++ mktemp
+ local LAST_ERR=/tmp/tmp.CrzyAxbnXD
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.odcyRAPA8g
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.CrzyAxbnXD
+ rm /tmp/tmp.odcyRAPA8g /tmp/tmp.CrzyAxbnXD
+ return 0
+ desc 'create PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PXC cluster
-----------------------------------------------------------------------------------

+ newpass=test-password
++ echo -n test-password
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZA==
+ cluster=some-name
+ spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/some-name.yml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/some-name.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.frxB8LPByO
++ mktemp
+ local LAST_ERR=/tmp/tmp.qIeCREcD9E
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.frxB8LPByO
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.qIeCREcD9E
+ rm /tmp/tmp.frxB8LPByO /tmp/tmp.qIeCREcD9E
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/client.yml
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/client.yml
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ local LAST_OUT=/tmp/tmp.FwiOhTd9m0
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1697-26afa527#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ local LAST_ERR=/tmp/tmp.G80dwGn8d0
+ local exit_status=0
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
++ seq 0 2
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-7894~
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.FwiOhTd9m0
deployment.apps/pxc-client created
+ cat /tmp/tmp.G80dwGn8d0
+ rm /tmp/tmp.FwiOhTd9m0 /tmp/tmp.G80dwGn8d0
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ some-name == \d\e\m\a\n\d\-\b\a\c\k\u\p ]]
+ [[ some-name == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/some-name.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/some-name.yml
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/some-name.yml
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ local LAST_OUT=/tmp/tmp.E9ls8vKIiP
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-7894~
++ mktemp
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ local LAST_ERR=/tmp/tmp.WmDsRpTLoW
+ local exit_status=0
++ seq 0 2
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1697-26afa527#'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.E9ls8vKIiP
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.WmDsRpTLoW
+ rm /tmp/tmp.E9ls8vKIiP /tmp/tmp.WmDsRpTLoW
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy some-name
++ local target_cluster=some-name
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tOXZcpJqn7
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xzHfOQfRod
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.tOXZcpJqn7
+++ cat /tmp/tmp.xzHfOQfRod
+++ rm /tmp/tmp.tOXZcpJqn7 /tmp/tmp.xzHfOQfRod
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.u1TCkSqgw8
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.eXgt2FO34r
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.u1TCkSqgw8
+++ cat /tmp/tmp.eXgt2FO34r
+++ rm /tmp/tmp.u1TCkSqgw8 /tmp/tmp.eXgt2FO34r
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo some-name-proxysql
++ return
+ local proxy=some-name-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-7894
++ mktemp
+ local LAST_OUT=/tmp/tmp.yLQL27rr9A
++ mktemp
+ local LAST_ERR=/tmp/tmp.TSZUfDfSUv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-7894
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-7894
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-7894
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.yLQL27rr9A
+ cat /tmp/tmp.TSZUfDfSUv
error: no matching resources found
+ rm /tmp/tmp.yLQL27rr9A /tmp/tmp.TSZUfDfSUv
+ return 1
+ true
+ wait_for_running some-name-proxysql 1
+ local name=some-name-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-proxysql-0 480
+ local pod=some-name-proxysql-0
+ local max_retry=480
+ local ns=
++ echo some-name-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
pod/some-name-proxysql-0 condition met
waiting for pod/some-name-proxysql-0 to become Ready.Ok
+ wait_for_running some-name-pxc 3
+ local name=some-name-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
waiting for pod/some-name-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-1 480
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
waiting for pod/some-name-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-2 480
+ local pod=some-name-pxc-2
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-2 condition met
waiting for pod/some-name-pxc-2 to become Ready.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
++ is_keyring_plugin_in_use some-name
++ local cluster=some-name
++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.knIyfTslq3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CkZnJFJn92
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.knIyfTslq3
++ cat /tmp/tmp.CkZnJFJn92
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.knIyfTslq3 /tmp/tmp.CkZnJFJn92
++ return 0
+ [[ -n '' ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-proxysql -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h some-name-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KyTv4c75V9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.88TodmduYW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KyTv4c75V9
++ cat /tmp/tmp.88TodmduYW
++ rm /tmp/tmp.KyTv4c75V9 /tmp/tmp.88TodmduYW
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-proxysql -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h some-name-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QdKrAsLo69
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wd5bNwcAtV
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.QdKrAsLo69
++ cat /tmp/tmp.wd5bNwcAtV
++ rm /tmp/tmp.QdKrAsLo69 /tmp/tmp.wd5bNwcAtV
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.J1KYNwXq9Z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.L61PM7XRpM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.J1KYNwXq9Z
++ cat /tmp/tmp.L61PM7XRpM
++ rm /tmp/tmp.J1KYNwXq9Z /tmp/tmp.L61PM7XRpM
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-1.sql /tmp/tmp.rI13iasBt2/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JAcdImd84s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KwRUGPl69h
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.JAcdImd84s
++ cat /tmp/tmp.KwRUGPl69h
++ rm /tmp/tmp.JAcdImd84s /tmp/tmp.KwRUGPl69h
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-1.sql /tmp/tmp.rI13iasBt2/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-1-57.sql ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6oqAQcMe6A
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AUWAWwT4wD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6oqAQcMe6A
++ cat /tmp/tmp.AUWAWwT4wD
++ rm /tmp/tmp.6oqAQcMe6A /tmp/tmp.AUWAWwT4wD
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-1.sql /tmp/tmp.rI13iasBt2/select-1.sql
++ is_keyring_plugin_in_use some-name
++ local cluster=some-name
++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4Nvm05qFKH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZTRQ3Cmt2Z
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4Nvm05qFKH
++ cat /tmp/tmp.ZTRQ3Cmt2Z
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.4Nvm05qFKH /tmp/tmp.ZTRQ3Cmt2Z
++ return 0
+ '[' '' ']'
+ desc 'test root'
+ set +o xtrace


-----------------------------------------------------------------------------------
test root
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets root dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=root
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.B4guOKiUX4
++ mktemp
+ local LAST_ERR=/tmp/tmp.dwnKNuJFFC
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.B4guOKiUX4
secret/my-cluster-secrets patched
+ cat /tmp/tmp.dwnKNuJFFC
+ rm /tmp/tmp.B4guOKiUX4 /tmp/tmp.dwnKNuJFFC
+ return 0
+ sleep 15
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4-57.sql ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CdsEVh2l1c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8zgKJhFPsx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.CdsEVh2l1c
++ cat /tmp/tmp.8zgKJhFPsx
++ rm /tmp/tmp.CdsEVh2l1c /tmp/tmp.8zgKJhFPsx
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4.sql /tmp/tmp.rI13iasBt2/select-4.sql
+ desc 'test proxyadmin'
+ set +o xtrace


-----------------------------------------------------------------------------------
test proxyadmin
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.k85PLNHcpV
++ mktemp
+ local LAST_ERR=/tmp/tmp.fshckNLEQQ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.k85PLNHcpV
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.fshckNLEQQ
+ rm /tmp/tmp.k85PLNHcpV /tmp/tmp.fshckNLEQQ
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c3pTXxTilg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.29mUOy7YEr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.c3pTXxTilg
++ cat /tmp/tmp.29mUOy7YEr
++ rm /tmp/tmp.c3pTXxTilg /tmp/tmp.29mUOy7YEr
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0apB7D1avT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.D0K1HQ4ASq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0apB7D1avT
++ cat /tmp/tmp.D0K1HQ4ASq
++ rm /tmp/tmp.0apB7D1avT /tmp/tmp.D0K1HQ4ASq
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uYx9rfVYdy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ulkLELbkU8
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.uYx9rfVYdy
++ cat /tmp/tmp.ulkLELbkU8
++ rm /tmp/tmp.uYx9rfVYdy /tmp/tmp.ulkLELbkU8
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.wEuKFT0jwe
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.9vd9noDWbJ
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.wEuKFT0jwe
+++++ cat /tmp/tmp.9vd9noDWbJ
+++++ rm /tmp/tmp.wEuKFT0jwe /tmp/tmp.9vd9noDWbJ
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.kJRpN48vHk
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.3ZWzKJLxX0
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.kJRpN48vHk
+++++ cat /tmp/tmp.3ZWzKJLxX0
+++++ rm /tmp/tmp.kJRpN48vHk /tmp/tmp.3ZWzKJLxX0
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SEG1bRMbbE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QInsa5nKtY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SEG1bRMbbE
++ cat /tmp/tmp.QInsa5nKtY
++ rm /tmp/tmp.SEG1bRMbbE /tmp/tmp.QInsa5nKtY
++ return 0
+ [[ 3 == \3 ]]
+ patch_secret my-cluster-secrets proxyadmin dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=proxyadmin
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.aMQLmxRHTR
++ mktemp
+ local LAST_ERR=/tmp/tmp.9a3qnL8oW8
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.aMQLmxRHTR
secret/my-cluster-secrets patched
+ cat /tmp/tmp.9a3qnL8oW8
+ rm /tmp/tmp.aMQLmxRHTR /tmp/tmp.9a3qnL8oW8
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qF0wifB0IC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cj8DsyLJe0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.qF0wifB0IC
++ cat /tmp/tmp.cj8DsyLJe0
++ rm /tmp/tmp.qF0wifB0IC /tmp/tmp.cj8DsyLJe0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TOmIqZju2G
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WcprttSJUg
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TOmIqZju2G
++ cat /tmp/tmp.WcprttSJUg
++ rm /tmp/tmp.TOmIqZju2G /tmp/tmp.WcprttSJUg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6JmxH1iYA9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gWYaLD0ovl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6JmxH1iYA9
++ cat /tmp/tmp.gWYaLD0ovl
++ rm /tmp/tmp.6JmxH1iYA9 /tmp/tmp.gWYaLD0ovl
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w8TuJRese0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5Hwmgpc3dO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.w8TuJRese0
++ cat /tmp/tmp.5Hwmgpc3dO
++ rm /tmp/tmp.w8TuJRese0 /tmp/tmp.5Hwmgpc3dO
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.B2hNmxBi55
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.dSFGMct1fh
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.B2hNmxBi55
+++++ cat /tmp/tmp.dSFGMct1fh
+++++ rm /tmp/tmp.B2hNmxBi55 /tmp/tmp.dSFGMct1fh
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.6KLxTKNDeR
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.XJcUS7pZQY
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.6KLxTKNDeR
+++++ cat /tmp/tmp.XJcUS7pZQY
+++++ rm /tmp/tmp.6KLxTKNDeR /tmp/tmp.XJcUS7pZQY
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ES5DaRkci1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.33Ch2d4zTV
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ES5DaRkci1
++ cat /tmp/tmp.33Ch2d4zTV
++ rm /tmp/tmp.ES5DaRkci1 /tmp/tmp.33Ch2d4zTV
++ return 0
+ [[ 3 == \3 ]]
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-0
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-0
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-2.sql /tmp/tmp.rI13iasBt2/select-2.sql
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-1
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-1
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-2.sql /tmp/tmp.rI13iasBt2/select-2.sql
+ compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 '' proxysql
+ local command_id=select-2
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-2
+ local postfix=
+ local container_name=proxysql
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 proxysql
+ local 'command=SHOW TABLES;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\'''
+ local pod=some-name-proxysql-2
+ local container_name=proxysql
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-2.sql /tmp/tmp.rI13iasBt2/select-2.sql
+ desc 'test xtrabackup'
+ set +o xtrace


-----------------------------------------------------------------------------------
test xtrabackup
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.elbc0HJejU
++ mktemp
+ local LAST_ERR=/tmp/tmp.d7tvtH0byi
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.elbc0HJejU
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.d7tvtH0byi
+ rm /tmp/tmp.elbc0HJejU /tmp/tmp.d7tvtH0byi
+ return 0
+ patch_secret my-cluster-secrets xtrabackup dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=xtrabackup
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.PMXjowhEZU
++ mktemp
+ local LAST_ERR=/tmp/tmp.jIvNQSz4au
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.PMXjowhEZU
secret/my-cluster-secrets patched
+ cat /tmp/tmp.jIvNQSz4au
+ rm /tmp/tmp.PMXjowhEZU /tmp/tmp.jIvNQSz4au
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pLk24tHI1z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.E4JilJZHQK
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pLk24tHI1z
++ cat /tmp/tmp.E4JilJZHQK
++ rm /tmp/tmp.pLk24tHI1z /tmp/tmp.E4JilJZHQK
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OXkwTprf7f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r3LKMJ1YLL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.OXkwTprf7f
++ cat /tmp/tmp.r3LKMJ1YLL
++ rm /tmp/tmp.OXkwTprf7f /tmp/tmp.r3LKMJ1YLL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AKg3YDbWgZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.avLynpbHtJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AKg3YDbWgZ
++ cat /tmp/tmp.avLynpbHtJ
++ rm /tmp/tmp.AKg3YDbWgZ /tmp/tmp.avLynpbHtJ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0R81VoJyFL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kEnInfFRtb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0R81VoJyFL
++ cat /tmp/tmp.kEnInfFRtb
++ rm /tmp/tmp.0R81VoJyFL /tmp/tmp.kEnInfFRtb
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AUDD3xVb1n
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VUgqBF8zqy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AUDD3xVb1n
++ cat /tmp/tmp.VUgqBF8zqy
++ rm /tmp/tmp.AUDD3xVb1n /tmp/tmp.VUgqBF8zqy
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VKzaoXOoki
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q71euzknGs
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VKzaoXOoki
++ cat /tmp/tmp.Q71euzknGs
++ rm /tmp/tmp.VKzaoXOoki /tmp/tmp.Q71euzknGs
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1dPGyuvEWk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iPy6ZI7RSE
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1dPGyuvEWk
++ cat /tmp/tmp.iPy6ZI7RSE
++ rm /tmp/tmp.1dPGyuvEWk /tmp/tmp.iPy6ZI7RSE
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uefrBYBJ64
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1EcCOT0EZ3
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.uefrBYBJ64
++ cat /tmp/tmp.1EcCOT0EZ3
++ rm /tmp/tmp.uefrBYBJ64 /tmp/tmp.1EcCOT0EZ3
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.2sye0laBee
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.rVjnVUoVGl
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.2sye0laBee
+++++ cat /tmp/tmp.rVjnVUoVGl
+++++ rm /tmp/tmp.2sye0laBee /tmp/tmp.rVjnVUoVGl
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.wPFyJ1ANPg
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.l9SYujyN5I
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.wPFyJ1ANPg
+++++ cat /tmp/tmp.l9SYujyN5I
+++++ rm /tmp/tmp.wPFyJ1ANPg /tmp/tmp.l9SYujyN5I
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b29bKRipGW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DuOAKdisp7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.b29bKRipGW
++ cat /tmp/tmp.DuOAKdisp7
++ rm /tmp/tmp.b29bKRipGW /tmp/tmp.DuOAKdisp7
++ return 0
+ [[ 2 == \2 ]]
+ compare_mysql_cmd_local select-3 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 '' pxc
+ local command_id=select-3
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\'''
+ local pod=some-name-pxc-0
+ local postfix=
+ local container_name=pxc
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ run_mysql_local 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 pxc
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\'''
+ local pod=some-name-pxc-0
+ local container_name=pxc
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-3.sql /tmp/tmp.rI13iasBt2/select-3.sql
+ desc 'test monitor'
+ set +o xtrace


-----------------------------------------------------------------------------------
test monitor
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=monitor
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.WMc1YPgQpE
++ mktemp
+ local LAST_ERR=/tmp/tmp.5j3QvZwSj5
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WMc1YPgQpE
secret/my-cluster-secrets patched
+ cat /tmp/tmp.5j3QvZwSj5
+ rm /tmp/tmp.WMc1YPgQpE /tmp/tmp.5j3QvZwSj5
+ return 0
+ wait_for_password_propagation my-cluster-secrets monitor
+ local secret=my-cluster-secrets
+ local user=monitor
+ local max_retry=240
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2rpx1k1oVp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wdjU1ep5uu
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2rpx1k1oVp
++ cat /tmp/tmp.wdjU1ep5uu
++ rm /tmp/tmp.2rpx1k1oVp /tmp/tmp.wdjU1ep5uu
++ return 0
+ local root_pass=test-password
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ echo 'Skipping dual password feature doesn'\''t work for 5.7. PXC 5.7 doesn'\''t support it!'
Skipping dual password feature doesn't work for 5.7. PXC 5.7 doesn't support it!
+ return
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2LRzpPa8xT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YWR4D3hEWX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2LRzpPa8xT
++ cat /tmp/tmp.YWR4D3hEWX
++ rm /tmp/tmp.2LRzpPa8xT /tmp/tmp.YWR4D3hEWX
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZQnqxZeRIz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uGGUlzvpDy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZQnqxZeRIz
++ cat /tmp/tmp.uGGUlzvpDy
++ rm /tmp/tmp.ZQnqxZeRIz /tmp/tmp.uGGUlzvpDy
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.T9GB0H6odh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5nig4kYrsG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.T9GB0H6odh
++ cat /tmp/tmp.5nig4kYrsG
++ rm /tmp/tmp.T9GB0H6odh /tmp/tmp.5nig4kYrsG
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.to4i8uFrQz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hL5WlIwD7R
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.to4i8uFrQz
++ cat /tmp/tmp.hL5WlIwD7R
++ rm /tmp/tmp.to4i8uFrQz /tmp/tmp.hL5WlIwD7R
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.oBSbYEVSSa
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.nOT69JplIq
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.oBSbYEVSSa
+++++ cat /tmp/tmp.nOT69JplIq
+++++ rm /tmp/tmp.oBSbYEVSSa /tmp/tmp.nOT69JplIq
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.EF1uy6bBWa
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.k0LyRoxSY8
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.EF1uy6bBWa
+++++ cat /tmp/tmp.k0LyRoxSY8
+++++ rm /tmp/tmp.EF1uy6bBWa /tmp/tmp.k0LyRoxSY8
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w2ylaf0kEs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N5BwRw3t2w
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.w2ylaf0kEs
++ cat /tmp/tmp.N5BwRw3t2w
++ rm /tmp/tmp.w2ylaf0kEs /tmp/tmp.N5BwRw3t2w
++ return 0
+ [[ 2 == \2 ]]
+ sleep 10
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4-57.sql ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7tb35HqreT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qr6RLamLAL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7tb35HqreT
++ cat /tmp/tmp.qr6RLamLAL
++ rm /tmp/tmp.7tb35HqreT /tmp/tmp.qr6RLamLAL
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4.sql /tmp/tmp.rI13iasBt2/select-4.sql
+ desc 'test operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
test operator
-----------------------------------------------------------------------------------

+ patch_secret my-cluster-secrets operator dGVzdC1wYXNzd29yZA==
+ local secret=my-cluster-secrets
+ local key=operator
+ local value=dGVzdC1wYXNzd29yZA==
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.py1EurvksU
++ mktemp
+ local LAST_ERR=/tmp/tmp.L8giYbqsg2
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.py1EurvksU
secret/my-cluster-secrets patched
+ cat /tmp/tmp.L8giYbqsg2
+ rm /tmp/tmp.py1EurvksU /tmp/tmp.L8giYbqsg2
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qTgXt0xE6H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XxPAhsyMy4
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.qTgXt0xE6H
++ cat /tmp/tmp.XxPAhsyMy4
++ rm /tmp/tmp.qTgXt0xE6H /tmp/tmp.XxPAhsyMy4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wAeFw5G9sJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qNfxsLQmgP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wAeFw5G9sJ
++ cat /tmp/tmp.qNfxsLQmgP
++ rm /tmp/tmp.wAeFw5G9sJ /tmp/tmp.qNfxsLQmgP
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KE2wcBsm0X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IUCkcSXJLR
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KE2wcBsm0X
++ cat /tmp/tmp.IUCkcSXJLR
++ rm /tmp/tmp.KE2wcBsm0X /tmp/tmp.IUCkcSXJLR
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.clheocTe26
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.LKGaKnrBXQ
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.clheocTe26
+++++ cat /tmp/tmp.LKGaKnrBXQ
+++++ rm /tmp/tmp.clheocTe26 /tmp/tmp.LKGaKnrBXQ
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.n9uhUD0v61
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.5TQHEE0FMk
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.n9uhUD0v61
+++++ cat /tmp/tmp.5TQHEE0FMk
+++++ rm /tmp/tmp.n9uhUD0v61 /tmp/tmp.5TQHEE0FMk
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZCbcRk5BCt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EsNdP6kRHS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZCbcRk5BCt
++ cat /tmp/tmp.EsNdP6kRHS
++ rm /tmp/tmp.ZCbcRk5BCt /tmp/tmp.EsNdP6kRHS
++ return 0
+ [[ 2 == \2 ]]
+ sleep 10
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4-57.sql ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.T3uA98QXml
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pVRz782RfZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.T3uA98QXml
++ cat /tmp/tmp.pVRz782RfZ
++ rm /tmp/tmp.T3uA98QXml /tmp/tmp.pVRz782RfZ
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4.sql /tmp/tmp.rI13iasBt2/select-4.sql
+ desc 'change secret name'
+ set +o xtrace


-----------------------------------------------------------------------------------
change secret name
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.LQMGoESAbl
++ mktemp
+ local LAST_ERR=/tmp/tmp.8YLcsNONbK
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.LQMGoESAbl
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.8YLcsNONbK
+ rm /tmp/tmp.LQMGoESAbl /tmp/tmp.8YLcsNONbK
+ return 0
+ sleep 30
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X5jp6ssQQq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iDgJ47J2LI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.X5jp6ssQQq
++ cat /tmp/tmp.iDgJ47J2LI
++ rm /tmp/tmp.X5jp6ssQQq /tmp/tmp.iDgJ47J2LI
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CRR2w0wz5g
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RjcUb5fBAL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.CRR2w0wz5g
++ cat /tmp/tmp.RjcUb5fBAL
++ rm /tmp/tmp.CRR2w0wz5g /tmp/tmp.RjcUb5fBAL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.s1FKHtv9pm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KWR5w1dV3F
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.s1FKHtv9pm
++ cat /tmp/tmp.KWR5w1dV3F
++ rm /tmp/tmp.s1FKHtv9pm /tmp/tmp.KWR5w1dV3F
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wespzU0vuH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aP0u6gH00i
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wespzU0vuH
++ cat /tmp/tmp.aP0u6gH00i
++ rm /tmp/tmp.wespzU0vuH /tmp/tmp.aP0u6gH00i
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IqOIBrnGoE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7F3So0e569
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IqOIBrnGoE
++ cat /tmp/tmp.7F3So0e569
++ rm /tmp/tmp.IqOIBrnGoE /tmp/tmp.7F3So0e569
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.APp0ohr1IB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IvyacaP73s
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.APp0ohr1IB
++ cat /tmp/tmp.IvyacaP73s
++ rm /tmp/tmp.APp0ohr1IB /tmp/tmp.IvyacaP73s
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UYJR9zPFUQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wszXhMtnKb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.UYJR9zPFUQ
++ cat /tmp/tmp.wszXhMtnKb
++ rm /tmp/tmp.UYJR9zPFUQ /tmp/tmp.wszXhMtnKb
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.asDuzEcIzr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yC6l9KjlA5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.asDuzEcIzr
++ cat /tmp/tmp.yC6l9KjlA5
++ rm /tmp/tmp.asDuzEcIzr /tmp/tmp.yC6l9KjlA5
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.PNttxZ2Nmd
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.ZUr5uRtaE5
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.PNttxZ2Nmd
+++++ cat /tmp/tmp.ZUr5uRtaE5
+++++ rm /tmp/tmp.PNttxZ2Nmd /tmp/tmp.ZUr5uRtaE5
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.QgkvLS611U
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.m69Y1HjPGZ
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.QgkvLS611U
+++++ cat /tmp/tmp.m69Y1HjPGZ
+++++ rm /tmp/tmp.QgkvLS611U /tmp/tmp.m69Y1HjPGZ
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Nh9vu26Ttj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vhhOF0A5PF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Nh9vu26Ttj
++ cat /tmp/tmp.vhhOF0A5PF
++ rm /tmp/tmp.Nh9vu26Ttj /tmp/tmp.vhhOF0A5PF
++ return 0
+ [[ 2 == \2 ]]
+ desc 'test new operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
test new operator
-----------------------------------------------------------------------------------

+ newpass=test-password2
++ echo -n test-password2
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZDI=
+ patch_secret my-cluster-secrets-2 operator dGVzdC1wYXNzd29yZDI=
+ local secret=my-cluster-secrets-2
+ local key=operator
+ local value=dGVzdC1wYXNzd29yZDI=
+ kubectl_bin patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.oJYVTEQZdV
++ mktemp
+ local LAST_ERR=/tmp/tmp.g6YnVPgx9r
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.oJYVTEQZdV
secret/my-cluster-secrets-2 patched
+ cat /tmp/tmp.g6YnVPgx9r
+ rm /tmp/tmp.oJYVTEQZdV /tmp/tmp.g6YnVPgx9r
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F8ZTEXLtFm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Mw1AgodSsY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.F8ZTEXLtFm
++ cat /tmp/tmp.Mw1AgodSsY
++ rm /tmp/tmp.F8ZTEXLtFm /tmp/tmp.Mw1AgodSsY
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KCeKfVkp4F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.w4zll7E9M9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KCeKfVkp4F
++ cat /tmp/tmp.w4zll7E9M9
++ rm /tmp/tmp.KCeKfVkp4F /tmp/tmp.w4zll7E9M9
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Bx4tnEh1Tb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hSaK2VkUKa
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Bx4tnEh1Tb
++ cat /tmp/tmp.hSaK2VkUKa
++ rm /tmp/tmp.Bx4tnEh1Tb /tmp/tmp.hSaK2VkUKa
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.GzjGeFXBxF
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.CeQBIsNSOK
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.GzjGeFXBxF
+++++ cat /tmp/tmp.CeQBIsNSOK
+++++ rm /tmp/tmp.GzjGeFXBxF /tmp/tmp.CeQBIsNSOK
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.8ZttcOuS3T
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.X05E6AufzL
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.8ZttcOuS3T
+++++ cat /tmp/tmp.X05E6AufzL
+++++ rm /tmp/tmp.8ZttcOuS3T /tmp/tmp.X05E6AufzL
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vUmkkACV3f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.awMRQ7oeLw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vUmkkACV3f
++ cat /tmp/tmp.awMRQ7oeLw
++ rm /tmp/tmp.vUmkkACV3f /tmp/tmp.awMRQ7oeLw
++ return 0
+ [[ 2 == \2 ]]
+ sleep 20
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4-57.sql ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sYZQuUaHUv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oHXxMA5Fkl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sYZQuUaHUv
++ cat /tmp/tmp.oHXxMA5Fkl
++ rm /tmp/tmp.sYZQuUaHUv /tmp/tmp.oHXxMA5Fkl
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4.sql /tmp/tmp.rI13iasBt2/select-4.sql
++ getSecretData my-cluster-secrets-2 root
++ local secretName=my-cluster-secrets-2
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets-2 '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RKosrPk3k5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RgsycAk9di
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets-2 '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RKosrPk3k5
++ cat /tmp/tmp.RgsycAk9di
++ rm /tmp/tmp.RKosrPk3k5 /tmp/tmp.RgsycAk9di
++ return 0
+ newpass='b)H_~wDqEgY<>n}<jH'
+ desc 'test new users sync'
+ set +o xtrace


-----------------------------------------------------------------------------------
test new users sync
-----------------------------------------------------------------------------------

+ run_mysql 'CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\''b)H_~wDqEgY<>n}<jH'\'';' '-h some-name-pxc -uroot -p'\''b)H_~wDqEgY<>n}<jH'\'''
+ local 'command=CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\''b)H_~wDqEgY<>n}<jH'\'';'
+ local 'uri=-h some-name-pxc -uroot -p'\''b)H_~wDqEgY<>n}<jH'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DIk1AUNURG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JCvRdSAyR2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DIk1AUNURG
++ cat /tmp/tmp.JCvRdSAyR2
++ rm /tmp/tmp.DIk1AUNURG /tmp/tmp.JCvRdSAyR2
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ sleep 40
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\''b)H_~wDqEgY<>n}<jH'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -utestsync -p'\''b)H_~wDqEgY<>n}<jH'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4-57.sql ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\''b)H_~wDqEgY<>n}<jH'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -utestsync -p'\''b)H_~wDqEgY<>n}<jH'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qRWtsYnHRz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eb9kfrGBuC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.qRWtsYnHRz
++ cat /tmp/tmp.eb9kfrGBuC
++ rm /tmp/tmp.qRWtsYnHRz /tmp/tmp.eb9kfrGBuC
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4.sql /tmp/tmp.rI13iasBt2/select-4.sql
++ getSecretData internal-some-name operator
++ local secretName=internal-some-name
++ local dataKey=operator
++ kubectl_bin get secrets/internal-some-name '--template={{.data.operator}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TlYaKDyYJf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SDY4XNfgpa
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/internal-some-name '--template={{.data.operator}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TlYaKDyYJf
++ cat /tmp/tmp.SDY4XNfgpa
++ rm /tmp/tmp.TlYaKDyYJf /tmp/tmp.SDY4XNfgpa
++ return 0
+ pass=test-password2
+ desc 'check secret without operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
check secret without operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.jb1NxfMj7v
++ mktemp
+ local LAST_ERR=/tmp/tmp.JFyso0pGaM
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.jb1NxfMj7v
secret/my-cluster-secrets-2 configured
+ cat /tmp/tmp.JFyso0pGaM
Warning: resource secrets/my-cluster-secrets-2 is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.jb1NxfMj7v /tmp/tmp.JFyso0pGaM
+ return 0
+ sleep 15
+ compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local command_id=select-4
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4-57.sql ]]
+ run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\'''
+ local 'command=SHOW TABLES;'
+ local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Us2qdxToK6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dPPIpSAb5i
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Us2qdxToK6
++ cat /tmp/tmp.dPPIpSAb5i
++ rm /tmp/tmp.Us2qdxToK6 /tmp/tmp.dPPIpSAb5i
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-4.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-4.sql /tmp/tmp.rI13iasBt2/select-4.sql
+ newpass=test-password2
++ echo -n test-password2
++ base64
+ newpassencrypted=dGVzdC1wYXNzd29yZDI=
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/conf/some-name.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#'
+ local LAST_OUT=/tmp/tmp.LQGjtEBM74
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1697-26afa527#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-7894~
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ mktemp
+ local LAST_ERR=/tmp/tmp.VduTlzgdH5
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.LQGjtEBM74
perconaxtradbcluster.pxc.percona.com/some-name configured
+ cat /tmp/tmp.VduTlzgdH5
+ rm /tmp/tmp.LQGjtEBM74 /tmp/tmp.VduTlzgdH5
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X3DZfYjvpB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9i4phEElQw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.X3DZfYjvpB
++ cat /tmp/tmp.9i4phEElQw
++ rm /tmp/tmp.X3DZfYjvpB /tmp/tmp.9i4phEElQw
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hxdohT6Cj6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zVU406kFMX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hxdohT6Cj6
++ cat /tmp/tmp.zVU406kFMX
++ rm /tmp/tmp.hxdohT6Cj6 /tmp/tmp.zVU406kFMX
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ltD5MjCOKT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cy4ezrcS5t
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ltD5MjCOKT
++ cat /tmp/tmp.cy4ezrcS5t
++ rm /tmp/tmp.ltD5MjCOKT /tmp/tmp.cy4ezrcS5t
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4UBLab0LFD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WfOrp3kXWC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4UBLab0LFD
++ cat /tmp/tmp.WfOrp3kXWC
++ rm /tmp/tmp.4UBLab0LFD /tmp/tmp.WfOrp3kXWC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fQweeqlzuD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y8upIyKKOz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.fQweeqlzuD
++ cat /tmp/tmp.y8upIyKKOz
++ rm /tmp/tmp.fQweeqlzuD /tmp/tmp.y8upIyKKOz
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hBT6YwOBc7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Cyf4wGBbAt
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.hBT6YwOBc7
++ cat /tmp/tmp.Cyf4wGBbAt
++ rm /tmp/tmp.hBT6YwOBc7 /tmp/tmp.Cyf4wGBbAt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LtkhJM8vTR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xb1B7zqFJN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LtkhJM8vTR
++ cat /tmp/tmp.xb1B7zqFJN
++ rm /tmp/tmp.LtkhJM8vTR /tmp/tmp.xb1B7zqFJN
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ccRFlHB2YB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6rKahujtG7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ccRFlHB2YB
++ cat /tmp/tmp.6rKahujtG7
++ rm /tmp/tmp.ccRFlHB2YB /tmp/tmp.6rKahujtG7
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.AMFOF9GqVj
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.V4aBpMPlDR
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.AMFOF9GqVj
+++++ cat /tmp/tmp.V4aBpMPlDR
+++++ rm /tmp/tmp.AMFOF9GqVj /tmp/tmp.V4aBpMPlDR
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0gOk7FCOob
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TxhxQYPssl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0gOk7FCOob
++ cat /tmp/tmp.TxhxQYPssl
++ rm /tmp/tmp.0gOk7FCOob /tmp/tmp.TxhxQYPssl
++ return 0
+ [[ 3 == \3 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ check_generation 2 haproxy some-name
+ local generation=2
+ local container=haproxy
+ local cluster=some-name
+ local current_generation
++ kubectl_bin get statefulset some-name-haproxy -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VbRawDQHRa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SNbdDdtsuL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-haproxy -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VbRawDQHRa
++ cat /tmp/tmp.SNbdDdtsuL
++ rm /tmp/tmp.VbRawDQHRa /tmp/tmp.SNbdDdtsuL
++ return 0
+ current_generation=2
+ [[ 2 != \2 ]]
+ patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZDI=
+ local secret=my-cluster-secrets
+ local key=monitor
+ local value=dGVzdC1wYXNzd29yZDI=
+ kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.MkgS6fPqin
++ mktemp
+ local LAST_ERR=/tmp/tmp.e46oBOJ5ty
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MkgS6fPqin
secret/my-cluster-secrets patched
+ cat /tmp/tmp.e46oBOJ5ty
+ rm /tmp/tmp.MkgS6fPqin /tmp/tmp.e46oBOJ5ty
+ return 0
+ sleep 15
+ wait_cluster_consistency some-name 3 3
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=3
+ '[' -z 3 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AvIyDbKuMD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wJwYSVoQyd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AvIyDbKuMD
++ cat /tmp/tmp.wJwYSVoQyd
++ rm /tmp/tmp.AvIyDbKuMD /tmp/tmp.wJwYSVoQyd
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dbdKLuB3lu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nBMg3vtTIF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.dbdKLuB3lu
++ cat /tmp/tmp.nBMg3vtTIF
++ rm /tmp/tmp.dbdKLuB3lu /tmp/tmp.nBMg3vtTIF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SidFmrJe2m
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HdoPaC609W
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SidFmrJe2m
++ cat /tmp/tmp.HdoPaC609W
++ rm /tmp/tmp.SidFmrJe2m /tmp/tmp.HdoPaC609W
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.M1OoRCgjmE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fRjVJjhbTH
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.M1OoRCgjmE
++ cat /tmp/tmp.fRjVJjhbTH
++ rm /tmp/tmp.M1OoRCgjmE /tmp/tmp.fRjVJjhbTH
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NXrF9x4tbc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9cZ4t1mMNz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NXrF9x4tbc
++ cat /tmp/tmp.9cZ4t1mMNz
++ rm /tmp/tmp.NXrF9x4tbc /tmp/tmp.9cZ4t1mMNz
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.vPGbARBmbL
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.fxS09VyEp3
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.vPGbARBmbL
+++++ cat /tmp/tmp.fxS09VyEp3
+++++ rm /tmp/tmp.vPGbARBmbL /tmp/tmp.fxS09VyEp3
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.odRo0fXdmw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZJYFA5dmjd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.odRo0fXdmw
++ cat /tmp/tmp.ZJYFA5dmjd
++ rm /tmp/tmp.odRo0fXdmw /tmp/tmp.ZJYFA5dmjd
++ return 0
+ [[ 3 == \3 ]]
+ compare_mysql_cmd select-3 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local command_id=select-3
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-3-57.sql ]]
+ run_mysql 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\'''
+ local 'command=SHOW DATABASES;'
+ local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\'''
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z1MpSdugna
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Kk4ThwvAWj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.z1MpSdugna
++ cat /tmp/tmp.Kk4ThwvAWj
++ rm /tmp/tmp.z1MpSdugna /tmp/tmp.Kk4ThwvAWj
++ return 0
+ client_pod=pxc-client-6db4b8c477-9f9n2
+ wait_pod pxc-client-6db4b8c477-9f9n2
+ local pod=pxc-client-6db4b8c477-9f9n2
+ local max_retry=480
+ local ns=
++ echo pxc-client-6db4b8c477-9f9n2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-6db4b8c477-9f9n2 condition met
waiting for pod/pxc-client-6db4b8c477-9f9n2 to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.rI13iasBt2/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/users/compare/select-3.sql /tmp/tmp.rI13iasBt2/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]]
+ check_generation 3 haproxy some-name
+ local generation=3
+ local container=haproxy
+ local cluster=some-name
+ local current_generation
++ kubectl_bin get statefulset some-name-haproxy -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.08Rf4yoawg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ORqAwuNnMI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-haproxy -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.08Rf4yoawg
++ cat /tmp/tmp.ORqAwuNnMI
++ rm /tmp/tmp.08Rf4yoawg /tmp/tmp.ORqAwuNnMI
++ return 0
+ current_generation=3
+ [[ 3 != \3 ]]
+ destroy users-7894
+ local namespace=users-7894
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ grep -v 'the object has been modified'
++ get_operator_pod
+ grep -v 'get backup status: Job.batch'
++ local label_prefix=app.kubernetes.io/
+ sort -u
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ tee /tmp/tmp.rI13iasBt2/operator.log
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+ grep -v level=info
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TjSb9CLSJa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rdAHkp15mZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TjSb9CLSJa
++ cat /tmp/tmp.rdAHkp15mZ
++ rm /tmp/tmp.TjSb9CLSJa /tmp/tmp.rdAHkp15mZ
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-67cbddd7f9-wzd8m
++ mktemp
+ local LAST_OUT=/tmp/tmp.aMWbsYD005
++ mktemp
+ local LAST_ERR=/tmp/tmp.u50QcV9jEl
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-67cbddd7f9-wzd8m
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.aMWbsYD005
+ cat /tmp/tmp.u50QcV9jEl
+ rm /tmp/tmp.aMWbsYD005 /tmp/tmp.u50QcV9jEl
+ return 0
2024-12-20T15:40:27.957Z	INFO	setup	Manager starting up	{"gitCommit": "26afa5272972c583e9dc3f39e65e29a2744c72d8", "gitBranch": "PR-1697-26afa527", "buildTime": "2024-12-20T13:32:57Z", "goVersion": "go1.23.4", "os": "linux", "arch": "amd64"}
2024-12-20T15:40:27.957Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.28.15-gke.1388000"}
2024-12-20T15:40:27.958Z	INFO	setup	Registering Components.
2024-12-20T15:40:31.043Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2024-12-20T15:40:31.047Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate
2024-12-20T15:40:31.047Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2024-12-20T15:40:31.047Z	INFO	controller-runtime.metrics	Starting metrics server
2024-12-20T15:40:31.047Z	INFO	controller-runtime.webhook	Starting webhook server
2024-12-20T15:40:31.047Z	INFO	setup	Starting the Cmd.
2024-12-20T15:40:31.047Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2024-12-20T15:40:31.048Z	INFO	controller-runtime.certwatcher	Starting certificate watcher
2024-12-20T15:40:31.048Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2024-12-20T15:40:31.148Z	INFO	attempting to acquire leader lease pxc-operator/08db1feb.percona.com...
2024-12-20T15:40:31.177Z	DEBUG	events	percona-xtradb-cluster-operator-67cbddd7f9-wzd8m_12af9778-35e1-4ec0-93be-c565f874915c became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"25ffdf56-d9e3-465c-902c-975ec55b3739","apiVersion":"coordination.k8s.io/v1","resourceVersion":"81854"}, "reason": "LeaderElection"}
2024-12-20T15:40:31.177Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2024-12-20T15:40:31.177Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2024-12-20T15:40:31.177Z	INFO	successfully acquired lease pxc-operator/08db1feb.percona.com
2024-12-20T15:40:31.178Z	INFO	Starting Controller	{"controller": "pxcbackup-controller"}
2024-12-20T15:40:31.178Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2024-12-20T15:40:31.178Z	INFO	Starting Controller	{"controller": "pxcrestore-controller"}
2024-12-20T15:40:31.178Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2024-12-20T15:40:31.409Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "worker count": 1}
2024-12-20T15:40:31.409Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2024-12-20T15:40:31.409Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "worker count": 1}
2024-12-20T15:40:57.441Z	INFO	Set CR version	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "115171cc-7348-4f4a-b8a1-b31d36a87b6a", "version": "1.16.0"}
2024-12-20T15:40:58.012Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "115171cc-7348-4f4a-b8a1-b31d36a87b6a", "object": "some-name-pxc"}
2024-12-20T15:40:58.187Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "115171cc-7348-4f4a-b8a1-b31d36a87b6a", "object": "some-name-pxc"}
2024-12-20T15:40:58.216Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "115171cc-7348-4f4a-b8a1-b31d36a87b6a", "object": "some-name-proxysql"}
2024-12-20T15:40:58.304Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "115171cc-7348-4f4a-b8a1-b31d36a87b6a", "object": "some-name-pxc"}
2024-12-20T15:40:58.368Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "115171cc-7348-4f4a-b8a1-b31d36a87b6a", "object": "some-name-pxc-unready"}
2024-12-20T15:40:58.601Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "115171cc-7348-4f4a-b8a1-b31d36a87b6a", "object": "some-name-proxysql"}
2024-12-20T15:40:58.870Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "115171cc-7348-4f4a-b8a1-b31d36a87b6a", "object": "some-name-proxysql-unready"}
2024-12-20T15:40:59.162Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "609a5e37-c116-4ff3-8b19-177f14c5eb8f", "object": "some-name-proxysql"}
2024-12-20T15:41:45.447Z	ERROR	failed to create db instance	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "85bbf8df-1790-4c3b-a75e-d21827daa0d0", "error": "dial tcp 10.245.120.88:33062: connect: connection refused"}
2024-12-20T15:41:45.995Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "85bbf8df-1790-4c3b-a75e-d21827daa0d0", "error": "reconcile users: retrieving pxc version: failed to reach any pod", "errorVerbose": "failed to reach any pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).mysqlVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:442\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:96\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nretrieving pxc version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile users\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:310\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:41:46.009Z	ERROR	failed to create db instance	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4389ff8a-975f-41b9-b88e-9e54106ddb29", "error": "dial tcp 10.245.120.88:33062: connect: connection refused"}
2024-12-20T15:41:46.039Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4389ff8a-975f-41b9-b88e-9e54106ddb29", "error": "reconcile users: retrieving pxc version: failed to reach any pod", "errorVerbose": "failed to reach any pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).mysqlVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:442\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:96\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nretrieving pxc version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile users\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:310\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:41:46.055Z	ERROR	failed to create db instance	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "82aead30-6c5b-450b-bef9-9c5ff23b83b5", "error": "dial tcp 10.245.120.88:33062: connect: connection refused"}
2024-12-20T15:41:46.084Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "82aead30-6c5b-450b-bef9-9c5ff23b83b5", "error": "reconcile users: retrieving pxc version: failed to reach any pod", "errorVerbose": "failed to reach any pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).mysqlVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:442\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:96\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nretrieving pxc version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile users\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:310\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:41:46.145Z	ERROR	failed to create db instance	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "0e6ad35a-0928-4178-8751-97731d12f4c7", "error": "dial tcp 10.245.120.88:33062: connect: connection refused"}
2024-12-20T15:41:46.168Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "0e6ad35a-0928-4178-8751-97731d12f4c7", "error": "reconcile users: retrieving pxc version: failed to reach any pod", "errorVerbose": "failed to reach any pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).mysqlVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:442\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:96\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nretrieving pxc version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile users\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:310\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:41:46.182Z	ERROR	failed to create db instance	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "889d8475-3098-47f2-a3e9-11cdb196fbb3", "error": "dial tcp 10.245.120.88:33062: connect: connection refused"}
2024-12-20T15:41:46.205Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "889d8475-3098-47f2-a3e9-11cdb196fbb3", "error": "reconcile users: retrieving pxc version: failed to reach any pod", "errorVerbose": "failed to reach any pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).mysqlVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:442\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:96\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nretrieving pxc version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile users\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:310\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:41:46.323Z	ERROR	failed to create db instance	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4d0d3f02-1c99-4d99-b186-97a097ec58b5", "error": "dial tcp 10.245.120.88:33062: connect: connection refused"}
2024-12-20T15:41:46.347Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4d0d3f02-1c99-4d99-b186-97a097ec58b5", "error": "reconcile users: retrieving pxc version: failed to reach any pod", "errorVerbose": "failed to reach any pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).mysqlVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:442\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:96\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nretrieving pxc version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile users\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:310\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:41:46.723Z	ERROR	failed to create db instance	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "c4f84acb-4a54-4225-9a31-431ac5a0c111", "error": "dial tcp 10.245.120.88:33062: connect: connection refused"}
2024-12-20T15:41:46.745Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "c4f84acb-4a54-4225-9a31-431ac5a0c111", "error": "reconcile users: retrieving pxc version: failed to reach any pod", "errorVerbose": "failed to reach any pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).mysqlVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:442\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:96\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nretrieving pxc version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile users\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:310\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:41:47.123Z	ERROR	failed to create db instance	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b13b9a8c-7b82-492a-a57a-b340469e7a8f", "error": "dial tcp 10.245.120.88:33062: connect: connection refused"}
2024-12-20T15:41:47.149Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b13b9a8c-7b82-492a-a57a-b340469e7a8f", "error": "reconcile users: retrieving pxc version: failed to reach any pod", "errorVerbose": "failed to reach any pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).mysqlVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:442\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:96\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nretrieving pxc version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile users\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:310\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:41:47.802Z	ERROR	failed to create db instance	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "eba64ece-18b1-45b6-befd-ffa31f7c3e2f", "error": "dial tcp 10.245.120.88:33062: connect: connection refused"}
2024-12-20T15:41:47.827Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "eba64ece-18b1-45b6-befd-ffa31f7c3e2f", "error": "reconcile users: retrieving pxc version: failed to reach any pod", "errorVerbose": "failed to reach any pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).mysqlVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:442\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:96\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nretrieving pxc version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile users\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:310\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:41:49.125Z	ERROR	failed to create db instance	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4882fdc8-40c9-470c-8375-af0f7f769796", "error": "dial tcp 10.245.120.88:33062: connect: connection refused"}
2024-12-20T15:41:49.184Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4882fdc8-40c9-470c-8375-af0f7f769796", "error": "reconcile users: retrieving pxc version: failed to reach any pod", "errorVerbose": "failed to reach any pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).mysqlVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:442\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:96\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nretrieving pxc version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile users\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:310\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:41:51.760Z	ERROR	failed to create db instance	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "588dc5a7-392e-4059-9931-73676a8b882e", "error": "dial tcp 10.245.120.88:33062: connect: connection refused"}
2024-12-20T15:41:51.789Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "588dc5a7-392e-4059-9931-73676a8b882e", "error": "reconcile users: retrieving pxc version: failed to reach any pod", "errorVerbose": "failed to reach any pod\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).mysqlVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:442\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:96\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nretrieving pxc version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile users\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:310\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:41:56.987Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "a85739da-73f0-4e11-a459-a629ca30f65a", "user": "operator"}
2024-12-20T15:41:57.021Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "a85739da-73f0-4e11-a459-a629ca30f65a", "user": "monitor"}
2024-12-20T15:41:57.075Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "a85739da-73f0-4e11-a459-a629ca30f65a"}
2024-12-20T15:41:57.122Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "a85739da-73f0-4e11-a459-a629ca30f65a", "user": "xtrabackup"}
2024-12-20T15:41:57.173Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "a85739da-73f0-4e11-a459-a629ca30f65a"}
2024-12-20T15:41:57.193Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "a85739da-73f0-4e11-a459-a629ca30f65a", "err": "get primary pxc pod: not found"}
2024-12-20T15:41:57.349Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "c583626d-1491-4c5a-a154-4f1161e5eecd", "err": "get primary pxc pod: not found"}
2024-12-20T15:44:07.701Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4f87f72c-6d8b-4bdb-bf74-6b92b75fdec2", "user": "root"}
2024-12-20T15:44:07.746Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4f87f72c-6d8b-4bdb-bf74-6b92b75fdec2", "user": "replication"}
2024-12-20T15:44:07.790Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4f87f72c-6d8b-4bdb-bf74-6b92b75fdec2", "new version": "5.7.44-48-57"}
2024-12-20T15:44:10.589Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4f87f72c-6d8b-4bdb-bf74-6b92b75fdec2"}
2024-12-20T15:44:15.396Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "281bf92d-297a-48ac-87d4-08ddcba0e4bf"}
2024-12-20T15:44:20.376Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "406a6b24-0323-4b51-9f64-552aa910341d"}
2024-12-20T15:44:25.491Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "228f78d9-471a-475f-b31e-5bc03e886a51"}
2024-12-20T15:44:31.384Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4c8e0cf8-e9c9-456c-9349-9cbf8b387340"}
2024-12-20T15:44:36.675Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f80c15d9-bfbc-48c4-a94c-594db69aea9e"}
2024-12-20T15:44:41.851Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "89f3195d-a5b8-4400-8489-6f615c8d1cdf"}
2024-12-20T15:44:46.874Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b6466b9c-7032-43ea-a114-d21e112fe667"}
2024-12-20T15:44:52.064Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "3c00b5a4-d8d8-49d7-a7d2-bf563fdc980d"}
2024-12-20T15:44:57.383Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "7599119f-cec3-4478-b180-808d26f9d2be"}
2024-12-20T15:45:02.462Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e9118d1e-b2a4-41b6-9db9-351772d8747a"}
2024-12-20T15:45:07.482Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "0b374206-2c3c-4c2a-8bf3-c8d2d7e8aded"}
2024-12-20T15:45:12.443Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "c81e3fe1-4b34-4ce3-b18e-a41e953d81e0"}
2024-12-20T15:45:17.783Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "17c90f49-4441-407e-8071-d74887717dd2"}
2024-12-20T15:45:20.393Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "604bdcb8-5d51-4631-83dc-9a4a96cb3d7a", "user": "root"}
2024-12-20T15:45:20.433Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "604bdcb8-5d51-4631-83dc-9a4a96cb3d7a", "user": "root"}
2024-12-20T15:45:20.443Z	INFO	MySQL init secret created	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "604bdcb8-5d51-4631-83dc-9a4a96cb3d7a", "secret": "some-name-mysql-init", "user": "root"}
2024-12-20T15:45:24.427Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "604bdcb8-5d51-4631-83dc-9a4a96cb3d7a"}
2024-12-20T15:45:24.438Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "604bdcb8-5d51-4631-83dc-9a4a96cb3d7a", "user": "root"}
2024-12-20T15:45:26.974Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "604bdcb8-5d51-4631-83dc-9a4a96cb3d7a"}
2024-12-20T15:45:32.167Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "94914089-bb53-42cb-b2ad-622048ef6f24"}
2024-12-20T15:45:36.363Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "2b5eafac-0625-4345-9a75-9a422eb3bcd4", "object": "some-name-proxysql"}
2024-12-20T15:45:37.353Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e13a9ba3-b1d8-42ea-a213-9461f6642918"}
2024-12-20T15:46:06.905Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e79635fd-9051-4446-a21c-926949415128", "error": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 /  / ERROR (line:515) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\nERROR (line:515) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\nERROR (line:515) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\nERROR (line:515) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\nERROR (line:515) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 /  / ERROR (line:515) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\nERROR (line:515) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\nERROR (line:515) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\nERROR (line:515) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\nERROR (line:515) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:925\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:46:19.101Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f9358f38-b738-44dc-bd22-268d6b541b20", "error": "syncusers: ERROR (line:515) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "syncusers: ERROR (line:515) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:928\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:46:25.409Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d623fdf6-35f1-4dfe-8be2-f60b3c32d7cc"}
2024-12-20T15:46:26.883Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "6d3ff4aa-cc0b-489a-89c1-37928ed24fe1", "user": "proxyadmin"}
2024-12-20T15:46:26.883Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "6d3ff4aa-cc0b-489a-89c1-37928ed24fe1", "user": "proxyadmin"}
2024-12-20T15:46:26.950Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "6d3ff4aa-cc0b-489a-89c1-37928ed24fe1", "user": "proxyadmin"}
2024-12-20T15:46:26.959Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "6d3ff4aa-cc0b-489a-89c1-37928ed24fe1", "user": "proxyadmin"}
2024-12-20T15:46:26.959Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "6d3ff4aa-cc0b-489a-89c1-37928ed24fe1", "last-applied-secret": "1a55867d0f4fa0d0643653b2eb57b2f690315d8d843a10599c9839c2a779a95f"}
2024-12-20T15:46:26.965Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "6d3ff4aa-cc0b-489a-89c1-37928ed24fe1", "object": "some-name-proxysql"}
2024-12-20T15:46:29.365Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "6d3ff4aa-cc0b-489a-89c1-37928ed24fe1", "error": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:925\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:47:12.662Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "15c97e72-d152-4c95-a76d-84d60a1f30a2"}
2024-12-20T15:47:17.874Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "07f5d9c0-e15d-4f29-8af1-f0328021e42a"}
2024-12-20T15:47:23.364Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "14e563b5-8969-4a12-aa81-bd763295ad2f"}
2024-12-20T15:47:28.276Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "fd50ee7e-4cdd-48f6-a4d2-806a9232710e"}
2024-12-20T15:47:33.497Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "6dc43719-cf68-48ce-80a2-12e46df2d23f"}
2024-12-20T15:47:35.123Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e9131ed5-a6ce-42e1-8655-5b5320b6edf3", "object": "some-name-proxysql"}
2024-12-20T15:47:35.171Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e9131ed5-a6ce-42e1-8655-5b5320b6edf3", "object": "some-name-proxysql"}
2024-12-20T15:47:41.527Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d85213ac-9e78-4158-98bf-f680235ee1bc", "user": "xtrabackup"}
2024-12-20T15:47:41.552Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d85213ac-9e78-4158-98bf-f680235ee1bc", "user": "xtrabackup"}
2024-12-20T15:47:41.559Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d85213ac-9e78-4158-98bf-f680235ee1bc", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-12-20T15:47:41.566Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d85213ac-9e78-4158-98bf-f680235ee1bc", "user": "xtrabackup"}
2024-12-20T15:47:41.566Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d85213ac-9e78-4158-98bf-f680235ee1bc", "last-applied-secret": "d11bcc70ca53ea410cd455e13d82e7225917b0861d9b677ed34c02c5f6fbaf1b"}
2024-12-20T15:47:41.572Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d85213ac-9e78-4158-98bf-f680235ee1bc", "object": "some-name-pxc"}
2024-12-20T15:47:45.732Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d85213ac-9e78-4158-98bf-f680235ee1bc"}
2024-12-20T15:49:19.252Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b2ea1ae9-e2c4-4ba8-9d74-3144ed42dd84", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-7894 on 10.134.16.10:53: no such host"}
2024-12-20T15:49:24.516Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "67c89906-f80f-48c6-ab8a-daf5c4b910d0", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-7894 on 10.134.16.10:53: no such host"}
2024-12-20T15:49:29.612Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "da8e32ad-a597-4e25-861b-b9dde825d693", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:49:34.683Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "cf58cf2a-7c91-44cf-a0f5-2443ac644bb1", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:49:39.850Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e1c59be4-b704-4b3b-bca9-44bb8fbf1505", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:49:44.922Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "6e31a7f8-b8ec-4f04-9d53-1c0fc136e6c7", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:49:50.035Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d1efd8cf-5519-4ff4-81c2-418c7c3b14d8", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:49:55.149Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "1df9c34f-55dc-4322-8215-c545468d66e9", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:50:03.193Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "2b067151-f49c-4202-9cf8-0029320f4aec"}
2024-12-20T15:50:08.098Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "5f156238-0e51-49a1-ad4f-51faf3d66ffd"}
2024-12-20T15:50:10.677Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "78944e2a-8c63-4e55-91a1-25e66123eeb1", "user": "monitor"}
2024-12-20T15:50:10.701Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "78944e2a-8c63-4e55-91a1-25e66123eeb1", "user": "monitor"}
2024-12-20T15:50:10.712Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "78944e2a-8c63-4e55-91a1-25e66123eeb1", "secret": "some-name-mysql-init", "user": "monitor"}
2024-12-20T15:50:10.754Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "78944e2a-8c63-4e55-91a1-25e66123eeb1", "user": "monitor"}
2024-12-20T15:50:10.766Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "78944e2a-8c63-4e55-91a1-25e66123eeb1", "user": "monitor"}
2024-12-20T15:50:10.766Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "78944e2a-8c63-4e55-91a1-25e66123eeb1", "last-applied-secret": "4e3282050745046d5044e0fcb5f2bf140d7320a1256a79c8ca2184f38f4f3098"}
2024-12-20T15:50:10.776Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "78944e2a-8c63-4e55-91a1-25e66123eeb1", "object": "some-name-proxysql"}
2024-12-20T15:50:14.397Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "78944e2a-8c63-4e55-91a1-25e66123eeb1", "error": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") /  / ", "errorVerbose": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") /  / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:925\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:50:43.699Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "859a51ce-8202-45e0-b64c-2fdfc00d907b"}
2024-12-20T15:50:59.971Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "24d0b1d4-ba07-487a-aa42-3428f90a09b0"}
2024-12-20T15:51:04.844Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "ee241627-8c25-4ceb-a5f6-145e031ef4b1"}
2024-12-20T15:51:09.949Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "5342e6fe-b21c-4cd3-afa8-6412fb6af3f7"}
2024-12-20T15:51:15.384Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "128a71b4-2932-4e3f-984f-9d21b872c780"}
2024-12-20T15:51:17.551Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "5e1c0e11-1ac6-449c-80de-71a3468079d5", "user": "operator"}
2024-12-20T15:51:17.579Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "5e1c0e11-1ac6-449c-80de-71a3468079d5", "user": "operator"}
2024-12-20T15:51:17.587Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "5e1c0e11-1ac6-449c-80de-71a3468079d5", "secret": "some-name-mysql-init", "user": "operator"}
2024-12-20T15:51:17.597Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "5e1c0e11-1ac6-449c-80de-71a3468079d5", "user": "operator"}
2024-12-20T15:51:17.597Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "5e1c0e11-1ac6-449c-80de-71a3468079d5", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-12-20T15:51:17.604Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "5e1c0e11-1ac6-449c-80de-71a3468079d5", "object": "some-name-proxysql"}
2024-12-20T15:51:22.328Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "5e1c0e11-1ac6-449c-80de-71a3468079d5", "error": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:925\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:51:47.907Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "3ecb828d-f603-457c-9614-8423e580c6fd"}
2024-12-20T15:51:51.189Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "1deb9947-8546-4b05-a862-10b59d1df75e"}
2024-12-20T15:51:56.073Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "5d833383-9740-43ae-901e-6f0823c9eec0"}
2024-12-20T15:52:01.450Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "9d654318-2c41-4ec1-85d4-7359332dfa1e"}
2024-12-20T15:52:06.495Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "7ce84650-5008-4b85-a96f-20ca8a30dd74"}
2024-12-20T15:52:11.898Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "5f0aa7df-596b-4bfe-8538-3ed1ab303342"}
2024-12-20T15:52:16.992Z	INFO	Created user secrets	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "secrets": "my-cluster-secrets-2"}
2024-12-20T15:52:16.993Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "root"}
2024-12-20T15:52:17.031Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "root"}
2024-12-20T15:52:17.042Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "secret": "some-name-mysql-init", "user": "root"}
2024-12-20T15:52:17.089Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "c34af81d-afb5-4981-bf3b-c011f13590c5"}
2024-12-20T15:52:20.784Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360"}
2024-12-20T15:52:20.810Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "root"}
2024-12-20T15:52:20.810Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "operator"}
2024-12-20T15:52:20.838Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "operator"}
2024-12-20T15:52:20.868Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "secret": "some-name-mysql-init", "user": "operator"}
2024-12-20T15:52:20.904Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "operator"}
2024-12-20T15:52:20.904Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "monitor"}
2024-12-20T15:52:20.929Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "monitor"}
2024-12-20T15:52:20.943Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "secret": "some-name-mysql-init", "user": "monitor"}
2024-12-20T15:52:20.984Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "monitor"}
2024-12-20T15:52:20.995Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "monitor"}
2024-12-20T15:52:20.995Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "xtrabackup"}
2024-12-20T15:52:21.018Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "xtrabackup"}
2024-12-20T15:52:21.031Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-12-20T15:52:21.042Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "xtrabackup"}
2024-12-20T15:52:21.042Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "replication"}
2024-12-20T15:52:21.067Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "replication"}
2024-12-20T15:52:21.082Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "secret": "some-name-mysql-init", "user": "replication"}
2024-12-20T15:52:21.094Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "replication"}
2024-12-20T15:52:21.094Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "proxyadmin"}
2024-12-20T15:52:21.137Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "proxyadmin"}
2024-12-20T15:52:21.147Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "user": "proxyadmin"}
2024-12-20T15:52:21.147Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "last-applied-secret": "1b5c70c4cd75d3768daa9ee061222189b80ce974fd3946eff1ed0f9467341f59"}
2024-12-20T15:52:21.147Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "last-applied-secret": "1b5c70c4cd75d3768daa9ee061222189b80ce974fd3946eff1ed0f9467341f59"}
2024-12-20T15:52:21.152Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "object": "some-name-pxc"}
2024-12-20T15:52:21.201Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "object": "some-name-proxysql"}
2024-12-20T15:52:23.265Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f2787b88-b141-471f-b478-bf7a3ed7a360", "error": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:925\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:54:04.695Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "fdad5fb9-9354-47e6-879a-7ea5d151e253", "err": "failed to connect to pod some-name-pxc-0: dial tcp 10.245.120.92:33062: connect: connection refused"}
2024-12-20T15:54:09.786Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "fa2d358c-619e-4da6-af93-a15a82fcc59a", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:54:14.867Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "23a847d2-dabd-42c7-a268-979c4b2d6a37", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:54:20.061Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "dab32dfa-1281-488c-9cef-f8c76426c42c", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:54:25.172Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f4c8a905-7d12-48c4-8049-78d4338edd4a", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:54:30.267Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "db965c6e-8bf4-4bdb-bf00-bb14e9108cd7", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:54:35.344Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "90e6d42f-cef4-4beb-acb2-b3f121b74c18", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:54:43.417Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "eb5c49c7-51dc-49dd-b0ad-496ba3e78931"}
2024-12-20T15:54:48.370Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "cde53fe8-731b-446b-8a71-116a9750b5c6"}
2024-12-20T15:54:53.775Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "03e1eed7-8007-44b0-84d1-4cd4b963fe21"}
2024-12-20T15:54:58.956Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d4f3dc76-67cb-4487-b694-983146eafea1"}
2024-12-20T15:55:01.496Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b595e5aa-aa1c-4cc0-a5bc-7efc268ee7f7", "user": "operator"}
2024-12-20T15:55:01.521Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b595e5aa-aa1c-4cc0-a5bc-7efc268ee7f7", "user": "operator"}
2024-12-20T15:55:01.533Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b595e5aa-aa1c-4cc0-a5bc-7efc268ee7f7", "secret": "some-name-mysql-init", "user": "operator"}
2024-12-20T15:55:01.549Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b595e5aa-aa1c-4cc0-a5bc-7efc268ee7f7", "user": "operator"}
2024-12-20T15:55:01.550Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b595e5aa-aa1c-4cc0-a5bc-7efc268ee7f7", "last-applied-secret": "048fd4cd34603d3d4faed46bd89e514735e32536d544fc375b8a99473e884e4f"}
2024-12-20T15:55:01.557Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b595e5aa-aa1c-4cc0-a5bc-7efc268ee7f7", "object": "some-name-proxysql"}
2024-12-20T15:55:06.416Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b595e5aa-aa1c-4cc0-a5bc-7efc268ee7f7", "error": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 /  / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-7894.svc.c' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:925\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:55:35.218Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "a79df260-3daf-45b5-9ab6-39f42119b584"}
2024-12-20T15:55:38.588Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "30586a28-d5f7-436f-8004-40321e16a483"}
2024-12-20T15:55:43.971Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b7189e33-1924-4979-b26b-fe76f428d8f4"}
2024-12-20T15:55:49.187Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "8b502a04-797e-44a2-bcfe-4398eda2b84d"}
2024-12-20T15:55:54.184Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b83021a2-cd50-4977-a28e-96b0d26f93d0"}
2024-12-20T15:55:59.671Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "337951bc-a0e4-4528-8871-0da2fb4d9d59"}
2024-12-20T15:56:04.592Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4fb2ff31-e29e-419c-90b1-b1bf8ada0c9f"}
2024-12-20T15:56:09.852Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "57a868fc-6953-4efa-8c7f-02d856d56b8e"}
2024-12-20T15:56:15.899Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "bac02fb3-ea57-4f7e-817c-dd3c0286dc95"}
2024-12-20T15:56:20.753Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "8a9cb35d-0502-4a2e-95b8-8dca18d2bf7c"}
2024-12-20T15:56:25.246Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "baa81f2f-3207-4df0-aea9-89579b8e613b"}
2024-12-20T15:56:31.257Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "ced02a2e-342d-4210-9efc-65064748ae6a"}
2024-12-20T15:56:35.751Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "05f150ba-2587-4594-8a92-f8293ef82d61"}
2024-12-20T15:56:40.861Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "88243d63-6008-400d-a471-37b17af8f84e"}
2024-12-20T15:56:46.588Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "387886ef-c478-488f-9426-2510642d5ebf"}
2024-12-20T15:56:51.495Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "dbfd7fa7-ddd4-44e5-ba8f-a93f0e96ad16"}
2024-12-20T15:56:57.068Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "0a63625e-afab-48c6-b68f-aa65c69ea262"}
2024-12-20T15:56:59.513Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "root"}
2024-12-20T15:56:59.554Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "root"}
2024-12-20T15:56:59.571Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "secret": "some-name-mysql-init", "user": "root"}
2024-12-20T15:57:03.422Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b"}
2024-12-20T15:57:03.429Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "root"}
2024-12-20T15:57:03.430Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "monitor"}
2024-12-20T15:57:03.457Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "monitor"}
2024-12-20T15:57:03.466Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "secret": "some-name-mysql-init", "user": "monitor"}
2024-12-20T15:57:03.508Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "monitor"}
2024-12-20T15:57:03.516Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "monitor"}
2024-12-20T15:57:03.516Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "xtrabackup"}
2024-12-20T15:57:03.540Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "xtrabackup"}
2024-12-20T15:57:03.548Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-12-20T15:57:03.558Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "xtrabackup"}
2024-12-20T15:57:03.558Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "proxyadmin"}
2024-12-20T15:57:03.602Z	INFO	Proxy user updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "proxyadmin"}
2024-12-20T15:57:03.611Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "user": "proxyadmin"}
2024-12-20T15:57:03.611Z	INFO	Proxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "last-applied-secret": "89a7cef9e9806272d5abb9c3169e497693fab2e1c37f3cfb1b43afbe3e5aa888"}
2024-12-20T15:57:03.611Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "last-applied-secret": "89a7cef9e9806272d5abb9c3169e497693fab2e1c37f3cfb1b43afbe3e5aa888"}
2024-12-20T15:57:03.646Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "object": "some-name-pxc"}
2024-12-20T15:57:03.740Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "object": "some-name-proxysql"}
2024-12-20T15:57:05.160Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "579076a6-f4e8-4df4-93ab-e4dd1b636e5b", "error": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 /  / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:925\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2024-12-20T15:58:47.195Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "8095723c-452e-4cd6-a987-4fe0d2c6e304", "err": "failed to connect to pod some-name-pxc-0: dial tcp 10.245.120.96:33062: connect: connection refused"}
2024-12-20T15:58:52.300Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "af46c5bf-a183-4c79-b885-62ad9e8638a0", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:58:57.453Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b0e284d2-9bd6-4b3e-b2fd-c24ac7df7644", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:59:02.538Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "b9b07f0e-f22e-4494-bb20-73fe807ba96b", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:59:07.613Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "3a4b662d-ccd2-46a2-8bcd-ca9a90808ced", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:59:12.745Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "f8dcf9a9-67ce-4435-99a0-63c6e1e70437", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:59:17.830Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "cfea3953-aaa7-431f-aa56-39c4a6e37480", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:59:22.898Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "7c4b4292-eec2-4760-b9b7-998f9b89fce8", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:59:27.976Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "4e20a198-391d-4c93-9f2f-7cfb1a610127", "primary name": "some-name-pxc-0.some-name-pxc.users-7894.svc.cluster.local"}
2024-12-20T15:59:33.647Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "root"}
2024-12-20T15:59:33.688Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "root"}
2024-12-20T15:59:33.698Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "secret": "some-name-mysql-init", "user": "root"}
2024-12-20T15:59:33.706Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "root"}
2024-12-20T15:59:33.706Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "operator"}
2024-12-20T15:59:33.730Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "operator"}
2024-12-20T15:59:33.739Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "secret": "some-name-mysql-init", "user": "operator"}
2024-12-20T15:59:33.745Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "operator"}
2024-12-20T15:59:33.746Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "monitor"}
2024-12-20T15:59:33.770Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "monitor"}
2024-12-20T15:59:33.779Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "secret": "some-name-mysql-init", "user": "monitor"}
2024-12-20T15:59:33.787Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "monitor"}
2024-12-20T15:59:33.787Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "xtrabackup"}
2024-12-20T15:59:33.812Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "xtrabackup"}
2024-12-20T15:59:33.821Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "secret": "some-name-mysql-init", "user": "xtrabackup"}
2024-12-20T15:59:33.831Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "xtrabackup"}
2024-12-20T15:59:33.831Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "replication"}
2024-12-20T15:59:33.854Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "replication"}
2024-12-20T15:59:33.864Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "secret": "some-name-mysql-init", "user": "replication"}
2024-12-20T15:59:33.872Z	INFO	HAProxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-12-20T15:59:33.872Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "user": "replication"}
2024-12-20T15:59:33.872Z	INFO	PXC pods will be restarted	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"}
2024-12-20T15:59:33.875Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "object": "some-name-pxc"}
2024-12-20T15:59:33.902Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "object": "some-name-haproxy"}
2024-12-20T15:59:33.941Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "object": "some-name-haproxy"}
2024-12-20T15:59:33.978Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "object": "some-name-pxc"}
2024-12-20T15:59:34.062Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "object": "some-name-haproxy"}
2024-12-20T15:59:34.243Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "object": "some-name-haproxy"}
2024-12-20T15:59:34.291Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "object": "some-name-haproxy-replicas"}
2024-12-20T15:59:37.582Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "e500ebd6-7d4d-4d11-8371-a23c004fa2eb", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.134.28.45:3306: connect: connection refused"}
2024-12-20T16:02:03.522Z	INFO	Password changed, updating user	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d29f4690-25f4-49d0-9ef7-f17e08bb9b69", "user": "monitor"}
2024-12-20T16:02:03.548Z	INFO	User password updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d29f4690-25f4-49d0-9ef7-f17e08bb9b69", "user": "monitor"}
2024-12-20T16:02:03.573Z	INFO	MySQL init secret updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d29f4690-25f4-49d0-9ef7-f17e08bb9b69", "secret": "some-name-mysql-init", "user": "monitor"}
2024-12-20T16:02:03.624Z	INFO	HAProxy pods will be restarted	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d29f4690-25f4-49d0-9ef7-f17e08bb9b69", "last-applied-secret": "81935ecf30ae523a6ee69582354f9488d47dbeb0f8c7c572af96870f2024284d"}
2024-12-20T16:02:03.624Z	INFO	Internal secrets updated	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d29f4690-25f4-49d0-9ef7-f17e08bb9b69", "user": "monitor"}
2024-12-20T16:02:03.629Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "users-7894", "name": "some-name", "reconcileID": "d29f4690-25f4-49d0-9ef7-f17e08bb9b69", "object": "some-name-haproxy"}
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).mysqlVersion
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileUsers
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:116
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:224
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:263
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:303
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.3/pkg/internal/controller/controller.go:316
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1027
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:308
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:96
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:427
[mysql] 2024/12/20 16:02:19 connection.go:49: unexpected EOF
[mysql] 2024/12/20 16:02:20 connection.go:49: unexpected EOF
[mysql] 2024/12/20 16:02:21 connection.go:49: unexpected EOF
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n users-7894 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.xDvh8trsII
++ mktemp
+ local LAST_ERR=/tmp/tmp.FeimSttgJ0
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xDvh8trsII
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.FeimSttgJ0
+ rm /tmp/tmp.xDvh8trsII /tmp/tmp.FeimSttgJ0
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.GgQLcCdeoZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.v8bsnhnDzK
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.GgQLcCdeoZ
No resources found
+ cat /tmp/tmp.v8bsnhnDzK
+ rm /tmp/tmp.GgQLcCdeoZ /tmp/tmp.v8bsnhnDzK
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.gu7Vr2rv1S
++ mktemp
+ local LAST_ERR=/tmp/tmp.n7brV9KWlr
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gu7Vr2rv1S
No resources found
+ cat /tmp/tmp.n7brV9KWlr
+ rm /tmp/tmp.gu7Vr2rv1S /tmp/tmp.n7brV9KWlr
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.XCcrkVBW8E
++ mktemp
+ local LAST_ERR=/tmp/tmp.VIvWCPwfog
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XCcrkVBW8E
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.VIvWCPwfog
+ rm /tmp/tmp.XCcrkVBW8E /tmp/tmp.VIvWCPwfog
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.16.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace users-7894
+ rm -rf /tmp/tmp.rI13iasBt2
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
+ local LAST_OUT=/tmp/tmp.W2lOgpmiVB
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.e6D3y1izFY
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.0b8RxcZRDG
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.43zBDjrhwR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace users-7894
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator