Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/logs/users-8-0.log WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1 + create_infra users-17620 + local ns=users-17620 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n users-7582 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.pxbFG876GT ++ mktemp + local LAST_ERR=/tmp/tmp.m6EXcqUaaU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pxbFG876GT perconaxtradbcluster.pxc.percona.com "some-name" deleted + cat /tmp/tmp.m6EXcqUaaU + rm /tmp/tmp.pxbFG876GT /tmp/tmp.m6EXcqUaaU + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.R60BIXq7HD ++ mktemp + local LAST_ERR=/tmp/tmp.mpRmEWkrCy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.R60BIXq7HD No resources found + cat /tmp/tmp.mpRmEWkrCy + rm /tmp/tmp.R60BIXq7HD /tmp/tmp.mpRmEWkrCy + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.MtL4mtGJEq ++ mktemp + local LAST_ERR=/tmp/tmp.UU5F3wuFr6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MtL4mtGJEq No resources found + cat /tmp/tmp.UU5F3wuFr6 + rm /tmp/tmp.MtL4mtGJEq /tmp/tmp.UU5F3wuFr6 + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' + '[' -n '' ']' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' ++ mktemp + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + awk '{print$1}' ++ mktemp + local LAST_OUT=/tmp/tmp.i1wW9QEKLY ++ mktemp + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + local LAST_OUT=/tmp/tmp.S2QQox0zmf + local LAST_ERR=/tmp/tmp.BDfQXG1esw + local exit_status=0 ++ mktemp ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.lEkLuzc9A8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.i1wW9QEKLY + cat /tmp/tmp.BDfQXG1esw + rm /tmp/tmp.i1wW9QEKLY /tmp/tmp.BDfQXG1esw + return 0 namespace "users-7582" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.S2QQox0zmf namespace "pxc-operator" deleted + cat /tmp/tmp.lEkLuzc9A8 + rm /tmp/tmp.S2QQox0zmf /tmp/tmp.lEkLuzc9A8 + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'namespace/pxc-operator - ' namespace/pxc-operator - + set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.h3wYfXUzqc ++ mktemp + local LAST_ERR=/tmp/tmp.T1FQu2S8lp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.h3wYfXUzqc namespace/pxc-operator created + cat /tmp/tmp.T1FQu2S8lp + rm /tmp/tmp.h3wYfXUzqc /tmp/tmp.T1FQu2S8lp + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.sZijBv9MxD +++ mktemp ++ local LAST_ERR=/tmp/tmp.uerUDopJ22 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sZijBv9MxD ++ cat /tmp/tmp.uerUDopJ22 ++ rm /tmp/tmp.sZijBv9MxD /tmp/tmp.uerUDopJ22 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1421-86089695-4-cluster9 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.k6mUWbYus1 ++ mktemp + local LAST_ERR=/tmp/tmp.w8iVYWRiXM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1421-86089695-4-cluster9 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.k6mUWbYus1 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1421-86089695-4-cluster9" modified. + cat /tmp/tmp.w8iVYWRiXM + rm /tmp/tmp.k6mUWbYus1 /tmp/tmp.w8iVYWRiXM + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.9aJzfmtBs5 ++ mktemp + local LAST_ERR=/tmp/tmp.yamjpIsXwq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9aJzfmtBs5 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.yamjpIsXwq + rm /tmp/tmp.9aJzfmtBs5 /tmp/tmp.yamjpIsXwq + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.mKUu8oM5pu ++ mktemp + local LAST_ERR=/tmp/tmp.h1Gg804AfC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mKUu8oM5pu clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.h1Gg804AfC + rm /tmp/tmp.mKUu8oM5pu /tmp/tmp.h1Gg804AfC + return 0 + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1421-86089695^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/deploy/cw-operator.yaml + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' ++ mktemp + local LAST_OUT=/tmp/tmp.QtnVmCOYYq ++ mktemp + local LAST_ERR=/tmp/tmp.bEP3W71qeo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QtnVmCOYYq deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.bEP3W71qeo + rm /tmp/tmp.QtnVmCOYYq /tmp/tmp.bEP3W71qeo + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.1TnAWRq5hS ++ mktemp + local LAST_ERR=/tmp/tmp.GDukvFHdkM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1TnAWRq5hS pod/percona-xtradb-cluster-operator-5c699b45b5-s9pfn condition met + cat /tmp/tmp.GDukvFHdkM + rm /tmp/tmp.1TnAWRq5hS /tmp/tmp.GDukvFHdkM + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.nqYlyTEBAX +++ mktemp ++ local LAST_ERR=/tmp/tmp.Tz6Tj0cKqc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nqYlyTEBAX ++ cat /tmp/tmp.Tz6Tj0cKqc ++ rm /tmp/tmp.nqYlyTEBAX /tmp/tmp.Tz6Tj0cKqc ++ return 0 + wait_pod percona-xtradb-cluster-operator-5c699b45b5-s9pfn 480 pxc-operator + local pod=percona-xtradb-cluster-operator-5c699b45b5-s9pfn + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-5c699b45b5-s9pfn ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-5c699b45b5-s9pfn condition met percona-xtradb-cluster-operator-5c699b45b5-s9pfn.Ok + sleep 3 + create_namespace users-17620 + local namespace=users-17620 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' + '[' -n '' ']' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces users-17620' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces users-17620 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace users-17620 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.E020qdBqMr + local LAST_OUT=/tmp/tmp.EldNWmudXH + xargs kubectl delete ns ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.SLYiefD7vc + local exit_status=0 + local LAST_ERR=/tmp/tmp.Nu6tqBGiqa + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace users-17620 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace users-17620 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.E020qdBqMr + cat /tmp/tmp.SLYiefD7vc + rm /tmp/tmp.E020qdBqMr /tmp/tmp.SLYiefD7vc + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace users-17620 Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.EldNWmudXH + cat /tmp/tmp.Nu6tqBGiqa Error from server (NotFound): namespaces "users-17620" not found + rm /tmp/tmp.EldNWmudXH /tmp/tmp.Nu6tqBGiqa + return 1 + : + wait_for_delete namespace/users-17620 + local res=namespace/users-17620 + echo -n 'namespace/users-17620 - ' namespace/users-17620 - + set +o xtrace Error from server (NotFound): namespaces "users-17620" not found + desc 'create namespace users-17620' + set +o xtrace ----------------------------------------------------------------------------------- create namespace users-17620 ----------------------------------------------------------------------------------- + kubectl_bin create namespace users-17620 ++ mktemp + local LAST_OUT=/tmp/tmp.JYy0XzFvm5 ++ mktemp + local LAST_ERR=/tmp/tmp.XU5JForCAy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace users-17620 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JYy0XzFvm5 namespace/users-17620 created + cat /tmp/tmp.XU5JForCAy + rm /tmp/tmp.JYy0XzFvm5 /tmp/tmp.XU5JForCAy + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.F3oWU5HbyD +++ mktemp ++ local LAST_ERR=/tmp/tmp.XUxzw6elb1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.F3oWU5HbyD ++ cat /tmp/tmp.XUxzw6elb1 ++ rm /tmp/tmp.F3oWU5HbyD /tmp/tmp.XUxzw6elb1 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1421-86089695-4-cluster9 --namespace=users-17620 ++ mktemp + local LAST_OUT=/tmp/tmp.wyygwYvbA3 ++ mktemp + local LAST_ERR=/tmp/tmp.fd3q2a3jU5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1421-86089695-4-cluster9 --namespace=users-17620 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wyygwYvbA3 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1421-86089695-4-cluster9" modified. + cat /tmp/tmp.fd3q2a3jU5 + rm /tmp/tmp.wyygwYvbA3 /tmp/tmp.fd3q2a3jU5 + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.LsQHwNoXMk ++ mktemp + local LAST_ERR=/tmp/tmp.JBpGt9AAsS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LsQHwNoXMk secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.JBpGt9AAsS + rm /tmp/tmp.LsQHwNoXMk /tmp/tmp.JBpGt9AAsS + return 0 + desc 'create PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC cluster ----------------------------------------------------------------------------------- + newpass=test-password ++ echo -n test-password ++ base64 + newpassencrypted=dGVzdC1wYXNzd29yZA== + cluster=some-name + spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/some-name.yml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/some-name.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ZNVdaJ2Vl1 ++ mktemp + local LAST_ERR=/tmp/tmp.NZPBVWppmh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZNVdaJ2Vl1 secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.NZPBVWppmh + rm /tmp/tmp.ZNVdaJ2Vl1 /tmp/tmp.NZPBVWppmh + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_OUT=/tmp/tmp.9xznhXriXZ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1421-86089695#' + local LAST_ERR=/tmp/tmp.QBviZr0fYJ + local exit_status=0 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-17620~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9xznhXriXZ deployment.apps/pxc-client created + cat /tmp/tmp.QBviZr0fYJ + rm /tmp/tmp.9xznhXriXZ /tmp/tmp.QBviZr0fYJ + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/conf/some-name.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1421-86089695#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_OUT=/tmp/tmp.3JQJ0IGhcb + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-17620~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ mktemp + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.gZ3GTt7DQ4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3JQJ0IGhcb perconaxtradbcluster.pxc.percona.com/some-name created + cat /tmp/tmp.gZ3GTt7DQ4 + rm /tmp/tmp.3JQJ0IGhcb /tmp/tmp.gZ3GTt7DQ4 + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DXOWFtUph1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FKUkcbm6H2 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.DXOWFtUph1 +++ cat /tmp/tmp.FKUkcbm6H2 +++ rm /tmp/tmp.DXOWFtUph1 /tmp/tmp.FKUkcbm6H2 +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QG2U1WkEsL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.aIkJ8FXQvY +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QG2U1WkEsL +++ cat /tmp/tmp.aIkJ8FXQvY +++ rm /tmp/tmp.QG2U1WkEsL /tmp/tmp.aIkJ8FXQvY +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo some-name-proxysql ++ return + local proxy=some-name-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-17620 ++ mktemp + local LAST_OUT=/tmp/tmp.a67zQC0pXh ++ mktemp + local LAST_ERR=/tmp/tmp.dArlt8Ca2Z + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-17620 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-17620 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-17620 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.a67zQC0pXh + cat /tmp/tmp.dArlt8Ca2Z error: no matching resources found + rm /tmp/tmp.a67zQC0pXh /tmp/tmp.dArlt8Ca2Z + return 1 + true + wait_for_running some-name-proxysql 1 + local name=some-name-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-proxysql-0 480 + local pod=some-name-proxysql-0 + local max_retry=480 + local ns= ++ echo some-name-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/some-name-proxysql-0 condition met some-name-proxysql-0.Ok + wait_for_running some-name-pxc 3 + local name=some-name-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ echo some-name-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met some-name-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-1 480 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met some-name-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-2 480 + local pod=some-name-pxc-2 + local max_retry=480 + local ns= ++ echo some-name-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-2 condition met some-name-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-proxysql -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h some-name-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lEBG6TX3Uw +++ mktemp ++ local LAST_ERR=/tmp/tmp.rKRhCA1Zfi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lEBG6TX3Uw ++ cat /tmp/tmp.rKRhCA1Zfi ++ rm /tmp/tmp.lEBG6TX3Uw /tmp/tmp.rKRhCA1Zfi ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-proxysql -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h some-name-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8BsB6QfM0M +++ mktemp ++ local LAST_ERR=/tmp/tmp.jEhAEeyjGx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8BsB6QfM0M ++ cat /tmp/tmp.jEhAEeyjGx ++ rm /tmp/tmp.8BsB6QfM0M /tmp/tmp.jEhAEeyjGx ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Jjqz76eKA8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.egZU3zNl2q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Jjqz76eKA8 ++ cat /tmp/tmp.egZU3zNl2q ++ rm /tmp/tmp.Jjqz76eKA8 /tmp/tmp.egZU3zNl2q ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-1.sql /tmp/tmp.TxLWXMXQ5Q/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0ypMDS5XLy +++ mktemp ++ local LAST_ERR=/tmp/tmp.r1Mh6u63ez ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0ypMDS5XLy ++ cat /tmp/tmp.r1Mh6u63ez ++ rm /tmp/tmp.0ypMDS5XLy /tmp/tmp.r1Mh6u63ez ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-1.sql /tmp/tmp.TxLWXMXQ5Q/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BJmMjyQZ6F +++ mktemp ++ local LAST_ERR=/tmp/tmp.c2ijS6aXg8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BJmMjyQZ6F ++ cat /tmp/tmp.c2ijS6aXg8 ++ rm /tmp/tmp.BJmMjyQZ6F /tmp/tmp.c2ijS6aXg8 ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-1.sql /tmp/tmp.TxLWXMXQ5Q/select-1.sql ++ is_keyring_plugin_in_use some-name ++ local cluster=some-name ++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HZD35EBW03 +++ mktemp ++ local LAST_ERR=/tmp/tmp.olUXwntCE5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HZD35EBW03 ++ cat /tmp/tmp.olUXwntCE5 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.HZD35EBW03 /tmp/tmp.olUXwntCE5 ++ return 0 + '[' '' ']' + desc 'test root' + set +o xtrace ----------------------------------------------------------------------------------- test root ----------------------------------------------------------------------------------- + patch_secret my-cluster-secrets root dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=root + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.cmqNacjk9Q ++ mktemp + local LAST_ERR=/tmp/tmp.DReVPKmybU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cmqNacjk9Q secret/my-cluster-secrets patched + cat /tmp/tmp.DReVPKmybU + rm /tmp/tmp.cmqNacjk9Q /tmp/tmp.DReVPKmybU + return 0 + sleep 15 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JZQbyBKDrv +++ mktemp ++ local LAST_ERR=/tmp/tmp.Peri3pLBby ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JZQbyBKDrv ++ cat /tmp/tmp.Peri3pLBby ++ rm /tmp/tmp.JZQbyBKDrv /tmp/tmp.Peri3pLBby ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.TxLWXMXQ5Q/select-4.sql + desc 'test proxyadmin' + set +o xtrace ----------------------------------------------------------------------------------- test proxyadmin ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.5aP6RJmi2X ++ mktemp + local LAST_ERR=/tmp/tmp.527CBlMxag + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5aP6RJmi2X perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.527CBlMxag + rm /tmp/tmp.5aP6RJmi2X /tmp/tmp.527CBlMxag + return 0 + sleep 15 + wait_cluster_consistency some-name 3 3 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=3 + '[' -z 3 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.J3KZEsRpdd +++ mktemp ++ local LAST_ERR=/tmp/tmp.RF2gtINhbB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.J3KZEsRpdd ++ cat /tmp/tmp.RF2gtINhbB ++ rm /tmp/tmp.J3KZEsRpdd /tmp/tmp.RF2gtINhbB ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZHVGOYKmcd +++ mktemp ++ local LAST_ERR=/tmp/tmp.Yzt7t43UGL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZHVGOYKmcd ++ cat /tmp/tmp.Yzt7t43UGL ++ rm /tmp/tmp.ZHVGOYKmcd /tmp/tmp.Yzt7t43UGL ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.6MxsqIkJlW ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.9ghzreOmK6 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.6MxsqIkJlW +++++ cat /tmp/tmp.9ghzreOmK6 +++++ rm /tmp/tmp.6MxsqIkJlW /tmp/tmp.9ghzreOmK6 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.OfZMftbPlZ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.nuzftQ85dd +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.OfZMftbPlZ +++++ cat /tmp/tmp.nuzftQ85dd +++++ rm /tmp/tmp.OfZMftbPlZ /tmp/tmp.nuzftQ85dd +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fCwo6gbauI +++ mktemp ++ local LAST_ERR=/tmp/tmp.E9ngDRzxFz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fCwo6gbauI ++ cat /tmp/tmp.E9ngDRzxFz ++ rm /tmp/tmp.fCwo6gbauI /tmp/tmp.E9ngDRzxFz ++ return 0 + [[ 3 == \3 ]] + patch_secret my-cluster-secrets proxyadmin dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=proxyadmin + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.ZVaEjHa5Uk ++ mktemp + local LAST_ERR=/tmp/tmp.54AWJCNG8A + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZVaEjHa5Uk secret/my-cluster-secrets patched + cat /tmp/tmp.54AWJCNG8A + rm /tmp/tmp.ZVaEjHa5Uk /tmp/tmp.54AWJCNG8A + return 0 + sleep 15 + wait_cluster_consistency some-name 3 3 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=3 + '[' -z 3 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gLNGJAH24G +++ mktemp ++ local LAST_ERR=/tmp/tmp.aBoMC3jfXe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gLNGJAH24G ++ cat /tmp/tmp.aBoMC3jfXe ++ rm /tmp/tmp.gLNGJAH24G /tmp/tmp.aBoMC3jfXe ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZytIaPDT6P +++ mktemp ++ local LAST_ERR=/tmp/tmp.EHgIE1t0tt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZytIaPDT6P ++ cat /tmp/tmp.EHgIE1t0tt ++ rm /tmp/tmp.ZytIaPDT6P /tmp/tmp.EHgIE1t0tt ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pkHSXwQLqq +++ mktemp ++ local LAST_ERR=/tmp/tmp.a2Ye72YNlQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pkHSXwQLqq ++ cat /tmp/tmp.a2Ye72YNlQ ++ rm /tmp/tmp.pkHSXwQLqq /tmp/tmp.a2Ye72YNlQ ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.53x533OmMd +++ mktemp ++ local LAST_ERR=/tmp/tmp.IALgCxU0CL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.53x533OmMd ++ cat /tmp/tmp.IALgCxU0CL ++ rm /tmp/tmp.53x533OmMd /tmp/tmp.IALgCxU0CL ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.WaB3IbATCi ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.pef4gQxWrb +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.WaB3IbATCi +++++ cat /tmp/tmp.pef4gQxWrb +++++ rm /tmp/tmp.WaB3IbATCi /tmp/tmp.pef4gQxWrb +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.NbG38lstZu ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.TbV07GmUob +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.NbG38lstZu +++++ cat /tmp/tmp.TbV07GmUob +++++ rm /tmp/tmp.NbG38lstZu /tmp/tmp.TbV07GmUob +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.headhzX2Xx +++ mktemp ++ local LAST_ERR=/tmp/tmp.kh1JT8h7Cv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.headhzX2Xx ++ cat /tmp/tmp.kh1JT8h7Cv ++ rm /tmp/tmp.headhzX2Xx /tmp/tmp.kh1JT8h7Cv ++ return 0 + [[ 3 == \3 ]] + compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 '' proxysql + local command_id=select-2 + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-0 + local postfix= + local container_name=proxysql + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-2-80.sql ']' + run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 proxysql + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-0 + local container_name=proxysql + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-2.sql /tmp/tmp.TxLWXMXQ5Q/select-2.sql + compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 '' proxysql + local command_id=select-2 + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-1 + local postfix= + local container_name=proxysql + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-2-80.sql ']' + run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 proxysql + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-1 + local container_name=proxysql + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-2.sql /tmp/tmp.TxLWXMXQ5Q/select-2.sql + compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 '' proxysql + local command_id=select-2 + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-2 + local postfix= + local container_name=proxysql + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-2-80.sql ']' + run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 proxysql + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-2 + local container_name=proxysql + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-2.sql /tmp/tmp.TxLWXMXQ5Q/select-2.sql + desc 'test xtrabackup' + set +o xtrace ----------------------------------------------------------------------------------- test xtrabackup ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.iqM1r1Z8Oc ++ mktemp + local LAST_ERR=/tmp/tmp.OFSImhHjDi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iqM1r1Z8Oc perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.OFSImhHjDi + rm /tmp/tmp.iqM1r1Z8Oc /tmp/tmp.OFSImhHjDi + return 0 + patch_secret my-cluster-secrets xtrabackup dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=xtrabackup + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.zyw1eP2CTT ++ mktemp + local LAST_ERR=/tmp/tmp.LNzzxQVZL9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zyw1eP2CTT secret/my-cluster-secrets patched + cat /tmp/tmp.LNzzxQVZL9 + rm /tmp/tmp.zyw1eP2CTT /tmp/tmp.LNzzxQVZL9 + return 0 + sleep 15 + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zQABgjPnuB +++ mktemp ++ local LAST_ERR=/tmp/tmp.BIdeTIZey8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zQABgjPnuB ++ cat /tmp/tmp.BIdeTIZey8 ++ rm /tmp/tmp.zQABgjPnuB /tmp/tmp.BIdeTIZey8 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2FByiqlbyx +++ mktemp ++ local LAST_ERR=/tmp/tmp.5ivmzItbLW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2FByiqlbyx ++ cat /tmp/tmp.5ivmzItbLW ++ rm /tmp/tmp.2FByiqlbyx /tmp/tmp.5ivmzItbLW ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.leRhzTV7N0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.fjuga5nX65 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.leRhzTV7N0 ++ cat /tmp/tmp.fjuga5nX65 ++ rm /tmp/tmp.leRhzTV7N0 /tmp/tmp.fjuga5nX65 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uFTQEYaZ9F +++ mktemp ++ local LAST_ERR=/tmp/tmp.Gkt13boGWs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uFTQEYaZ9F ++ cat /tmp/tmp.Gkt13boGWs ++ rm /tmp/tmp.uFTQEYaZ9F /tmp/tmp.Gkt13boGWs ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7JTyJd2Mwz +++ mktemp ++ local LAST_ERR=/tmp/tmp.GXl4YjdI6I ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7JTyJd2Mwz ++ cat /tmp/tmp.GXl4YjdI6I ++ rm /tmp/tmp.7JTyJd2Mwz /tmp/tmp.GXl4YjdI6I ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xMsON8ckx7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.txmCFeTGSO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xMsON8ckx7 ++ cat /tmp/tmp.txmCFeTGSO ++ rm /tmp/tmp.xMsON8ckx7 /tmp/tmp.txmCFeTGSO ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vUjqOY4D1a +++ mktemp ++ local LAST_ERR=/tmp/tmp.EW7Ruq8kWs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vUjqOY4D1a ++ cat /tmp/tmp.EW7Ruq8kWs ++ rm /tmp/tmp.vUjqOY4D1a /tmp/tmp.EW7Ruq8kWs ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hL9ivLshkx +++ mktemp ++ local LAST_ERR=/tmp/tmp.dOvuGvErnc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hL9ivLshkx ++ cat /tmp/tmp.dOvuGvErnc ++ rm /tmp/tmp.hL9ivLshkx /tmp/tmp.dOvuGvErnc ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Jy6AznyhJD +++ mktemp ++ local LAST_ERR=/tmp/tmp.4O2Uko3SpB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Jy6AznyhJD ++ cat /tmp/tmp.4O2Uko3SpB ++ rm /tmp/tmp.Jy6AznyhJD /tmp/tmp.4O2Uko3SpB ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Z88wH5NTbq ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ydDVvXBCC8 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Z88wH5NTbq +++++ cat /tmp/tmp.ydDVvXBCC8 +++++ rm /tmp/tmp.Z88wH5NTbq /tmp/tmp.ydDVvXBCC8 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ItkudVHbHF ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.sezjbhR5Jc +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ItkudVHbHF +++++ cat /tmp/tmp.sezjbhR5Jc +++++ rm /tmp/tmp.ItkudVHbHF /tmp/tmp.sezjbhR5Jc +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.b0CBsSMwPe +++ mktemp ++ local LAST_ERR=/tmp/tmp.6okqovWdD3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.b0CBsSMwPe ++ cat /tmp/tmp.6okqovWdD3 ++ rm /tmp/tmp.b0CBsSMwPe /tmp/tmp.6okqovWdD3 ++ return 0 + [[ 2 == \2 ]] + compare_mysql_cmd_local select-3 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 '' pxc + local command_id=select-3 + local 'command=SHOW DATABASES;' + local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' + local pod=some-name-pxc-0 + local postfix= + local container_name=pxc + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-3-80.sql ']' + run_mysql_local 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 pxc + local 'command=SHOW DATABASES;' + local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' + local pod=some-name-pxc-0 + local container_name=pxc + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-3.sql /tmp/tmp.TxLWXMXQ5Q/select-3.sql + desc 'test monitor' + set +o xtrace ----------------------------------------------------------------------------------- test monitor ----------------------------------------------------------------------------------- + patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=monitor + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.qvZKaAnHZ4 ++ mktemp + local LAST_ERR=/tmp/tmp.cEKvAQdM7m + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qvZKaAnHZ4 secret/my-cluster-secrets patched + cat /tmp/tmp.cEKvAQdM7m + rm /tmp/tmp.qvZKaAnHZ4 /tmp/tmp.cEKvAQdM7m + return 0 + wait_for_password_propagation my-cluster-secrets monitor + local secret=my-cluster-secrets + local user=monitor + local max_retry=240 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.c4FzWz0yCr +++ mktemp ++ local LAST_ERR=/tmp/tmp.mwsGIfO6QU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.c4FzWz0yCr ++ cat /tmp/tmp.mwsGIfO6QU ++ rm /tmp/tmp.c4FzWz0yCr /tmp/tmp.mwsGIfO6QU ++ return 0 + local root_pass=test-password + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + retry=0 + is_password_updated monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep additional_password ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.m8VfxAL6Hf +++ mktemp ++ local LAST_ERR=/tmp/tmp.IzU3tuy8mt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.m8VfxAL6Hf ++ cat /tmp/tmp.IzU3tuy8mt ++ rm /tmp/tmp.m8VfxAL6Hf /tmp/tmp.IzU3tuy8mt ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + echo 'waiting for password update' waiting for password update + sleep 1 + let retry+=1 + [[ 1 -ge 240 ]] + is_password_updated monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep additional_password + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kXYvIxCnsl +++ mktemp ++ local LAST_ERR=/tmp/tmp.qgprncoD5W ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kXYvIxCnsl ++ cat /tmp/tmp.qgprncoD5W ++ rm /tmp/tmp.kXYvIxCnsl /tmp/tmp.qgprncoD5W ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace {"additional_password": "*C1F414D9BAF378B656A849B31F9F8AF3125F558B"} + retry=0 + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cg1o7dtDnC +++ mktemp ++ local LAST_ERR=/tmp/tmp.HBjE33pfV7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cg1o7dtDnC ++ cat /tmp/tmp.HBjE33pfV7 ++ rm /tmp/tmp.cg1o7dtDnC /tmp/tmp.HBjE33pfV7 ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 1 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rpv6z5Hk2S +++ mktemp ++ local LAST_ERR=/tmp/tmp.xHTfM3locG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rpv6z5Hk2S ++ cat /tmp/tmp.xHTfM3locG ++ rm /tmp/tmp.rpv6z5Hk2S /tmp/tmp.xHTfM3locG ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 2 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3muJZsNKU6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.KAbEpFKvAV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3muJZsNKU6 ++ cat /tmp/tmp.KAbEpFKvAV ++ rm /tmp/tmp.3muJZsNKU6 /tmp/tmp.KAbEpFKvAV ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 3 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1RRr8IP0MD +++ mktemp ++ local LAST_ERR=/tmp/tmp.ugtK8kX3oe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1RRr8IP0MD ++ cat /tmp/tmp.ugtK8kX3oe ++ rm /tmp/tmp.1RRr8IP0MD /tmp/tmp.ugtK8kX3oe ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 4 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DoBbhnIR8J +++ mktemp ++ local LAST_ERR=/tmp/tmp.EV6fH1mafi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DoBbhnIR8J ++ cat /tmp/tmp.EV6fH1mafi ++ rm /tmp/tmp.DoBbhnIR8J /tmp/tmp.EV6fH1mafi ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 5 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + grep NULL + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1S054rhItL +++ mktemp ++ local LAST_ERR=/tmp/tmp.yQTybuqVJm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1S054rhItL ++ cat /tmp/tmp.yQTybuqVJm ++ rm /tmp/tmp.1S054rhItL /tmp/tmp.yQTybuqVJm ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 6 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + grep NULL + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jZ5HOQhSFQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.eACIvmxNr8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jZ5HOQhSFQ ++ cat /tmp/tmp.eACIvmxNr8 ++ rm /tmp/tmp.jZ5HOQhSFQ /tmp/tmp.eACIvmxNr8 ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-65d95667fc-5hskv ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 7 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + grep NULL + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NlPqHASpvd +++ mktemp ++ local LAST_ERR=/tmp/tmp.lqGZiM4Z1J ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NlPqHASpvd ++ cat /tmp/tmp.lqGZiM4Z1J ++ rm /tmp/tmp.NlPqHASpvd /tmp/tmp.lqGZiM4Z1J ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 8 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8ESo2bmvwI +++ mktemp ++ local LAST_ERR=/tmp/tmp.KwYqX496NT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8ESo2bmvwI ++ cat /tmp/tmp.KwYqX496NT ++ rm /tmp/tmp.8ESo2bmvwI /tmp/tmp.KwYqX496NT ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 9 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MMmL9oHcKh +++ mktemp ++ local LAST_ERR=/tmp/tmp.9AilhUuKUF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MMmL9oHcKh ++ cat /tmp/tmp.9AilhUuKUF ++ rm /tmp/tmp.MMmL9oHcKh /tmp/tmp.9AilhUuKUF ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 10 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FdNASLQ3zy +++ mktemp ++ local LAST_ERR=/tmp/tmp.AGNo6mnaAX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FdNASLQ3zy ++ cat /tmp/tmp.AGNo6mnaAX ++ rm /tmp/tmp.FdNASLQ3zy /tmp/tmp.AGNo6mnaAX ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace NULL + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.swCl61gTZf +++ mktemp ++ local LAST_ERR=/tmp/tmp.Vx1GF8ebGp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.swCl61gTZf ++ cat /tmp/tmp.Vx1GF8ebGp ++ rm /tmp/tmp.swCl61gTZf /tmp/tmp.Vx1GF8ebGp ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rzWuSdgiPy +++ mktemp ++ local LAST_ERR=/tmp/tmp.THcGapUmMl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rzWuSdgiPy ++ cat /tmp/tmp.THcGapUmMl ++ rm /tmp/tmp.rzWuSdgiPy /tmp/tmp.THcGapUmMl ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ahcUPyNtCh ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.06odBsLg5d +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ahcUPyNtCh +++++ cat /tmp/tmp.06odBsLg5d +++++ rm /tmp/tmp.ahcUPyNtCh /tmp/tmp.06odBsLg5d +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.fr9b3ST2ZQ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.h2LdncSJLP +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.fr9b3ST2ZQ +++++ cat /tmp/tmp.h2LdncSJLP +++++ rm /tmp/tmp.fr9b3ST2ZQ /tmp/tmp.h2LdncSJLP +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K2jD3MsPkZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.jS0s3gcR1a ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.K2jD3MsPkZ ++ cat /tmp/tmp.jS0s3gcR1a ++ rm /tmp/tmp.K2jD3MsPkZ /tmp/tmp.jS0s3gcR1a ++ return 0 + [[ 2 == \2 ]] + sleep 10 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.b1z6ZtPiK9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.8VUEDnzTyB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.b1z6ZtPiK9 ++ cat /tmp/tmp.8VUEDnzTyB ++ rm /tmp/tmp.b1z6ZtPiK9 /tmp/tmp.8VUEDnzTyB ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.TxLWXMXQ5Q/select-4.sql + desc 'test operator' + set +o xtrace ----------------------------------------------------------------------------------- test operator ----------------------------------------------------------------------------------- + patch_secret my-cluster-secrets operator dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=operator + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.p6tXxxAhRO ++ mktemp + local LAST_ERR=/tmp/tmp.IqGB971X0J + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.p6tXxxAhRO secret/my-cluster-secrets patched + cat /tmp/tmp.IqGB971X0J + rm /tmp/tmp.p6tXxxAhRO /tmp/tmp.IqGB971X0J + return 0 + sleep 15 + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kIX66yGNu2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.EEASaEEKnp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kIX66yGNu2 ++ cat /tmp/tmp.EEASaEEKnp ++ rm /tmp/tmp.kIX66yGNu2 /tmp/tmp.EEASaEEKnp ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8fVYY8orR8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.KAVZLRcCrV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8fVYY8orR8 ++ cat /tmp/tmp.KAVZLRcCrV ++ rm /tmp/tmp.8fVYY8orR8 /tmp/tmp.KAVZLRcCrV ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wOd17jiSci +++ mktemp ++ local LAST_ERR=/tmp/tmp.eBzpSGoH1L ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wOd17jiSci ++ cat /tmp/tmp.eBzpSGoH1L ++ rm /tmp/tmp.wOd17jiSci /tmp/tmp.eBzpSGoH1L ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.cqun607ZYB ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.p8ybLfPtww +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.cqun607ZYB +++++ cat /tmp/tmp.p8ybLfPtww +++++ rm /tmp/tmp.cqun607ZYB /tmp/tmp.p8ybLfPtww +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.HNF1h50RKx ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.NcbBy8wl5y +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.HNF1h50RKx +++++ cat /tmp/tmp.NcbBy8wl5y +++++ rm /tmp/tmp.HNF1h50RKx /tmp/tmp.NcbBy8wl5y +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cdUQlYYknv +++ mktemp ++ local LAST_ERR=/tmp/tmp.iWuXaU5ewq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cdUQlYYknv ++ cat /tmp/tmp.iWuXaU5ewq ++ rm /tmp/tmp.cdUQlYYknv /tmp/tmp.iWuXaU5ewq ++ return 0 + [[ 2 == \2 ]] + sleep 10 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bQArM6LrxE +++ mktemp ++ local LAST_ERR=/tmp/tmp.d8R2GXge4L ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bQArM6LrxE ++ cat /tmp/tmp.d8R2GXge4L ++ rm /tmp/tmp.bQArM6LrxE /tmp/tmp.d8R2GXge4L ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.TxLWXMXQ5Q/select-4.sql + desc 'change secret name' + set +o xtrace ----------------------------------------------------------------------------------- change secret name ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.i9tucyIxI0 ++ mktemp + local LAST_ERR=/tmp/tmp.xxkJPEoUVu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.i9tucyIxI0 perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.xxkJPEoUVu + rm /tmp/tmp.i9tucyIxI0 /tmp/tmp.xxkJPEoUVu + return 0 + sleep 30 + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aIk7qJms40 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7uhpdjsYbr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aIk7qJms40 ++ cat /tmp/tmp.7uhpdjsYbr ++ rm /tmp/tmp.aIk7qJms40 /tmp/tmp.7uhpdjsYbr ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pqdCdtxu7m +++ mktemp ++ local LAST_ERR=/tmp/tmp.7UMmPbUOQD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pqdCdtxu7m ++ cat /tmp/tmp.7UMmPbUOQD ++ rm /tmp/tmp.pqdCdtxu7m /tmp/tmp.7UMmPbUOQD ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pftXvvAqbO +++ mktemp ++ local LAST_ERR=/tmp/tmp.S0042VN1mZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pftXvvAqbO ++ cat /tmp/tmp.S0042VN1mZ ++ rm /tmp/tmp.pftXvvAqbO /tmp/tmp.S0042VN1mZ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aI91WtfIOW +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z67PkIia2J ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aI91WtfIOW ++ cat /tmp/tmp.Z67PkIia2J ++ rm /tmp/tmp.aI91WtfIOW /tmp/tmp.Z67PkIia2J ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4yHoGNxUGo +++ mktemp ++ local LAST_ERR=/tmp/tmp.C4ZYBX6TUe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4yHoGNxUGo ++ cat /tmp/tmp.C4ZYBX6TUe ++ rm /tmp/tmp.4yHoGNxUGo /tmp/tmp.C4ZYBX6TUe ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wUwQasZNAU +++ mktemp ++ local LAST_ERR=/tmp/tmp.XFfFcyBA3p ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wUwQasZNAU ++ cat /tmp/tmp.XFfFcyBA3p ++ rm /tmp/tmp.wUwQasZNAU /tmp/tmp.XFfFcyBA3p ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Jo07SdjClY +++ mktemp ++ local LAST_ERR=/tmp/tmp.G4jTmu2sr8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Jo07SdjClY ++ cat /tmp/tmp.G4jTmu2sr8 ++ rm /tmp/tmp.Jo07SdjClY /tmp/tmp.G4jTmu2sr8 ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w89bq7WV4P +++ mktemp ++ local LAST_ERR=/tmp/tmp.6295VchlPn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w89bq7WV4P ++ cat /tmp/tmp.6295VchlPn ++ rm /tmp/tmp.w89bq7WV4P /tmp/tmp.6295VchlPn ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.4Y941cgf6S ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.mDZt0mYMSc +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.4Y941cgf6S +++++ cat /tmp/tmp.mDZt0mYMSc +++++ rm /tmp/tmp.4Y941cgf6S /tmp/tmp.mDZt0mYMSc +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.zK8fzRQYd1 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.3ftEU2SNr4 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.zK8fzRQYd1 +++++ cat /tmp/tmp.3ftEU2SNr4 +++++ rm /tmp/tmp.zK8fzRQYd1 /tmp/tmp.3ftEU2SNr4 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yqd1KkaFXi +++ mktemp ++ local LAST_ERR=/tmp/tmp.4we9Bs6hmG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yqd1KkaFXi ++ cat /tmp/tmp.4we9Bs6hmG ++ rm /tmp/tmp.yqd1KkaFXi /tmp/tmp.4we9Bs6hmG ++ return 0 + [[ 2 == \2 ]] + desc 'test new operator' + set +o xtrace ----------------------------------------------------------------------------------- test new operator ----------------------------------------------------------------------------------- + newpass=test-password2 ++ echo -n test-password2 ++ base64 + newpassencrypted=dGVzdC1wYXNzd29yZDI= + patch_secret my-cluster-secrets-2 operator dGVzdC1wYXNzd29yZDI= + local secret=my-cluster-secrets-2 + local key=operator + local value=dGVzdC1wYXNzd29yZDI= + kubectl_bin patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.bUPAQsEIbn ++ mktemp + local LAST_ERR=/tmp/tmp.eKUYMGeLFH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bUPAQsEIbn secret/my-cluster-secrets-2 patched + cat /tmp/tmp.eKUYMGeLFH + rm /tmp/tmp.bUPAQsEIbn /tmp/tmp.eKUYMGeLFH + return 0 + sleep 15 + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jUypLICZRP +++ mktemp ++ local LAST_ERR=/tmp/tmp.5qhRqsINrj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jUypLICZRP ++ cat /tmp/tmp.5qhRqsINrj ++ rm /tmp/tmp.jUypLICZRP /tmp/tmp.5qhRqsINrj ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5XTdXyc75H +++ mktemp ++ local LAST_ERR=/tmp/tmp.BSAkNsNpNC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5XTdXyc75H ++ cat /tmp/tmp.BSAkNsNpNC ++ rm /tmp/tmp.5XTdXyc75H /tmp/tmp.BSAkNsNpNC ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ThiYoqe2XC +++ mktemp ++ local LAST_ERR=/tmp/tmp.fZqesh7DVl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ThiYoqe2XC ++ cat /tmp/tmp.fZqesh7DVl ++ rm /tmp/tmp.ThiYoqe2XC /tmp/tmp.fZqesh7DVl ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A1ozvEKcKO +++ mktemp ++ local LAST_ERR=/tmp/tmp.7lVBsoYK4B ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A1ozvEKcKO ++ cat /tmp/tmp.7lVBsoYK4B ++ rm /tmp/tmp.A1ozvEKcKO /tmp/tmp.7lVBsoYK4B ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.DXcnM223WR ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.MA5ZIxJliM +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.DXcnM223WR +++++ cat /tmp/tmp.MA5ZIxJliM +++++ rm /tmp/tmp.DXcnM223WR /tmp/tmp.MA5ZIxJliM +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.lxpU3Vcy32 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.CfyWDDDtK1 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.lxpU3Vcy32 +++++ cat /tmp/tmp.CfyWDDDtK1 +++++ rm /tmp/tmp.lxpU3Vcy32 /tmp/tmp.CfyWDDDtK1 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fCbrdRSROI +++ mktemp ++ local LAST_ERR=/tmp/tmp.kyMgrlPGfe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fCbrdRSROI ++ cat /tmp/tmp.kyMgrlPGfe ++ rm /tmp/tmp.fCbrdRSROI /tmp/tmp.kyMgrlPGfe ++ return 0 + [[ 2 == \2 ]] + sleep 20 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8uBMp2V6Kb +++ mktemp ++ local LAST_ERR=/tmp/tmp.rWEd87vOWC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8uBMp2V6Kb ++ cat /tmp/tmp.rWEd87vOWC ++ rm /tmp/tmp.8uBMp2V6Kb /tmp/tmp.rWEd87vOWC ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.TxLWXMXQ5Q/select-4.sql ++ getSecretData my-cluster-secrets-2 root ++ local secretName=my-cluster-secrets-2 ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets-2 '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.CyuWhUrFHX +++ mktemp ++ local LAST_ERR=/tmp/tmp.zvyR4E5L6o ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets-2 '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CyuWhUrFHX ++ cat /tmp/tmp.zvyR4E5L6o ++ rm /tmp/tmp.CyuWhUrFHX /tmp/tmp.zvyR4E5L6o ++ return 0 + newpass=',]={gW^pbSJDa{Jna' + desc 'test new users sync' + set +o xtrace ----------------------------------------------------------------------------------- test new users sync ----------------------------------------------------------------------------------- + run_mysql 'CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\'',]={gW^pbSJDa{Jna'\'';' '-h some-name-pxc -uroot -p'\'',]={gW^pbSJDa{Jna'\''' + local 'command=CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\'',]={gW^pbSJDa{Jna'\'';' + local 'uri=-h some-name-pxc -uroot -p'\'',]={gW^pbSJDa{Jna'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.djbSMebUTw +++ mktemp ++ local LAST_ERR=/tmp/tmp.7GSjoAShnP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.djbSMebUTw ++ cat /tmp/tmp.7GSjoAShnP ++ rm /tmp/tmp.djbSMebUTw /tmp/tmp.7GSjoAShnP ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + sleep 40 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\'',]={gW^pbSJDa{Jna'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -utestsync -p'\'',]={gW^pbSJDa{Jna'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\'',]={gW^pbSJDa{Jna'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -utestsync -p'\'',]={gW^pbSJDa{Jna'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Lm5O0nXfz1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.L9y8paamuT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Lm5O0nXfz1 ++ cat /tmp/tmp.L9y8paamuT ++ rm /tmp/tmp.Lm5O0nXfz1 /tmp/tmp.L9y8paamuT ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.TxLWXMXQ5Q/select-4.sql ++ getSecretData internal-some-name operator ++ local secretName=internal-some-name ++ local dataKey=operator ++ kubectl_bin get secrets/internal-some-name '--template={{.data.operator}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.lS13gn9p05 +++ mktemp ++ local LAST_ERR=/tmp/tmp.g4zNBDXzhX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/internal-some-name '--template={{.data.operator}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lS13gn9p05 ++ cat /tmp/tmp.g4zNBDXzhX ++ rm /tmp/tmp.lS13gn9p05 /tmp/tmp.g4zNBDXzhX ++ return 0 + pass=test-password2 + desc 'check secret without operator' + set +o xtrace ----------------------------------------------------------------------------------- check secret without operator ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.uoUPK2LegF ++ mktemp + local LAST_ERR=/tmp/tmp.DPOKUPWoIT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uoUPK2LegF secret/my-cluster-secrets-2 configured + cat /tmp/tmp.DPOKUPWoIT Warning: resource secrets/my-cluster-secrets-2 is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.uoUPK2LegF /tmp/tmp.DPOKUPWoIT + return 0 + sleep 15 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pxq8bSZmtD +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZNqYWq40ii ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Pxq8bSZmtD ++ cat /tmp/tmp.ZNqYWq40ii ++ rm /tmp/tmp.Pxq8bSZmtD /tmp/tmp.ZNqYWq40ii ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.TxLWXMXQ5Q/select-4.sql + newpass=test-password2 ++ echo -n test-password2 ++ base64 + newpassencrypted=dGVzdC1wYXNzd29yZDI= + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/conf/some-name.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/conf/some-name.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/conf/some-name.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1421-86089695#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-17620~ + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.KivMCX0kSL ++ mktemp + local LAST_ERR=/tmp/tmp.xj8cEZ4qJE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KivMCX0kSL perconaxtradbcluster.pxc.percona.com/some-name configured + cat /tmp/tmp.xj8cEZ4qJE + rm /tmp/tmp.KivMCX0kSL /tmp/tmp.xj8cEZ4qJE + return 0 + sleep 15 + wait_cluster_consistency some-name 3 3 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=3 + '[' -z 3 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pveScqd57M +++ mktemp ++ local LAST_ERR=/tmp/tmp.BA3sx4VD31 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pveScqd57M ++ cat /tmp/tmp.BA3sx4VD31 ++ rm /tmp/tmp.pveScqd57M /tmp/tmp.BA3sx4VD31 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rbZ3C7VDjI +++ mktemp ++ local LAST_ERR=/tmp/tmp.tAuPCHEeb9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rbZ3C7VDjI ++ cat /tmp/tmp.tAuPCHEeb9 ++ rm /tmp/tmp.rbZ3C7VDjI /tmp/tmp.tAuPCHEeb9 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eKtj4ctt83 +++ mktemp ++ local LAST_ERR=/tmp/tmp.S0yX7DPjzS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eKtj4ctt83 ++ cat /tmp/tmp.S0yX7DPjzS ++ rm /tmp/tmp.eKtj4ctt83 /tmp/tmp.S0yX7DPjzS ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.egLInHoHnD +++ mktemp ++ local LAST_ERR=/tmp/tmp.PaD7lvSK1T ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.egLInHoHnD ++ cat /tmp/tmp.PaD7lvSK1T ++ rm /tmp/tmp.egLInHoHnD /tmp/tmp.PaD7lvSK1T ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VRJccOTtTW +++ mktemp ++ local LAST_ERR=/tmp/tmp.21tH387VgW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VRJccOTtTW ++ cat /tmp/tmp.21tH387VgW ++ rm /tmp/tmp.VRJccOTtTW /tmp/tmp.21tH387VgW ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dtHUwjcKMZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.aC38XxBME4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dtHUwjcKMZ ++ cat /tmp/tmp.aC38XxBME4 ++ rm /tmp/tmp.dtHUwjcKMZ /tmp/tmp.aC38XxBME4 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pbGyKkDXjE +++ mktemp ++ local LAST_ERR=/tmp/tmp.Gw2INyY8Wa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pbGyKkDXjE ++ cat /tmp/tmp.Gw2INyY8Wa ++ rm /tmp/tmp.pbGyKkDXjE /tmp/tmp.Gw2INyY8Wa ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H8mayabnBU +++ mktemp ++ local LAST_ERR=/tmp/tmp.JWZKiB6git ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.H8mayabnBU ++ cat /tmp/tmp.JWZKiB6git ++ rm /tmp/tmp.H8mayabnBU /tmp/tmp.JWZKiB6git ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PARsMbfidP +++ mktemp ++ local LAST_ERR=/tmp/tmp.aMeTQaEKus ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PARsMbfidP ++ cat /tmp/tmp.aMeTQaEKus ++ rm /tmp/tmp.PARsMbfidP /tmp/tmp.aMeTQaEKus ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.278e9Rxukb +++ mktemp ++ local LAST_ERR=/tmp/tmp.qRgkTx6WJp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.278e9Rxukb ++ cat /tmp/tmp.qRgkTx6WJp ++ rm /tmp/tmp.278e9Rxukb /tmp/tmp.qRgkTx6WJp ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ndZxATG0PY +++ mktemp ++ local LAST_ERR=/tmp/tmp.J5BOffxgoj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ndZxATG0PY ++ cat /tmp/tmp.J5BOffxgoj ++ rm /tmp/tmp.ndZxATG0PY /tmp/tmp.J5BOffxgoj ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JH5jTSkZA3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.3QxlsJG1PI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JH5jTSkZA3 ++ cat /tmp/tmp.3QxlsJG1PI ++ rm /tmp/tmp.JH5jTSkZA3 /tmp/tmp.3QxlsJG1PI ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aEFvfemwV6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Kdewy8P3zz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aEFvfemwV6 ++ cat /tmp/tmp.Kdewy8P3zz ++ rm /tmp/tmp.aEFvfemwV6 /tmp/tmp.Kdewy8P3zz ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IpQ2p8MHDb +++ mktemp ++ local LAST_ERR=/tmp/tmp.F74lngADe8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IpQ2p8MHDb ++ cat /tmp/tmp.F74lngADe8 ++ rm /tmp/tmp.IpQ2p8MHDb /tmp/tmp.F74lngADe8 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZcuHqK7tOP +++ mktemp ++ local LAST_ERR=/tmp/tmp.rNrcFInz9J ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZcuHqK7tOP ++ cat /tmp/tmp.rNrcFInz9J ++ rm /tmp/tmp.ZcuHqK7tOP /tmp/tmp.rNrcFInz9J ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FC1xy7VykL +++ mktemp ++ local LAST_ERR=/tmp/tmp.J5awxFnMeo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FC1xy7VykL ++ cat /tmp/tmp.J5awxFnMeo ++ rm /tmp/tmp.FC1xy7VykL /tmp/tmp.J5awxFnMeo ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.StzmHmTD4t +++ mktemp ++ local LAST_ERR=/tmp/tmp.5xptVIiKvz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.StzmHmTD4t ++ cat /tmp/tmp.5xptVIiKvz ++ rm /tmp/tmp.StzmHmTD4t /tmp/tmp.5xptVIiKvz ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.C6T9OJyhw3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.aFIY8DTSUv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.C6T9OJyhw3 ++ cat /tmp/tmp.aFIY8DTSUv ++ rm /tmp/tmp.C6T9OJyhw3 /tmp/tmp.aFIY8DTSUv ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.BZRniNn1p9 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.qrvui93BF2 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.BZRniNn1p9 +++++ cat /tmp/tmp.qrvui93BF2 +++++ rm /tmp/tmp.BZRniNn1p9 /tmp/tmp.qrvui93BF2 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-haproxy ++++ return +++ local cluster_proxy=some-name-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HcUTeG3wM5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.TvybAnr257 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HcUTeG3wM5 ++ cat /tmp/tmp.TvybAnr257 ++ rm /tmp/tmp.HcUTeG3wM5 /tmp/tmp.TvybAnr257 ++ return 0 + [[ 3 == \3 ]] + patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZDI= + local secret=my-cluster-secrets + local key=monitor + local value=dGVzdC1wYXNzd29yZDI= + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.9U3yt2gRkS ++ mktemp + local LAST_ERR=/tmp/tmp.SI10K4n3oa + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9U3yt2gRkS secret/my-cluster-secrets patched + cat /tmp/tmp.SI10K4n3oa + rm /tmp/tmp.9U3yt2gRkS /tmp/tmp.SI10K4n3oa + return 0 + sleep 15 + wait_cluster_consistency some-name 3 3 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=3 + '[' -z 3 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fPBSw51aeV +++ mktemp ++ local LAST_ERR=/tmp/tmp.roqXSJBZxN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fPBSw51aeV ++ cat /tmp/tmp.roqXSJBZxN ++ rm /tmp/tmp.fPBSw51aeV /tmp/tmp.roqXSJBZxN ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HSjVXoUnt8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FKyGIkTJyN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HSjVXoUnt8 ++ cat /tmp/tmp.FKyGIkTJyN ++ rm /tmp/tmp.HSjVXoUnt8 /tmp/tmp.FKyGIkTJyN ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cvb580FR1A +++ mktemp ++ local LAST_ERR=/tmp/tmp.tQFvsCUaTA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cvb580FR1A ++ cat /tmp/tmp.tQFvsCUaTA ++ rm /tmp/tmp.cvb580FR1A /tmp/tmp.tQFvsCUaTA ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MMf3ovipc3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.e9UEJCniJV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MMf3ovipc3 ++ cat /tmp/tmp.e9UEJCniJV ++ rm /tmp/tmp.MMf3ovipc3 /tmp/tmp.e9UEJCniJV ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.c6QD4lXqOX +++ mktemp ++ local LAST_ERR=/tmp/tmp.yHAOYgWvfl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.c6QD4lXqOX ++ cat /tmp/tmp.yHAOYgWvfl ++ rm /tmp/tmp.c6QD4lXqOX /tmp/tmp.yHAOYgWvfl ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Q6EwhEfA0v +++ mktemp ++ local LAST_ERR=/tmp/tmp.Tt1CXPxEzf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Q6EwhEfA0v ++ cat /tmp/tmp.Tt1CXPxEzf ++ rm /tmp/tmp.Q6EwhEfA0v /tmp/tmp.Tt1CXPxEzf ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.BXpyqk68To ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.asmOP8aJx5 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.BXpyqk68To +++++ cat /tmp/tmp.asmOP8aJx5 +++++ rm /tmp/tmp.BXpyqk68To /tmp/tmp.asmOP8aJx5 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-haproxy ++++ return +++ local cluster_proxy=some-name-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.l5wnKc8vO0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Dk1uz7aF9w ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.l5wnKc8vO0 ++ cat /tmp/tmp.Dk1uz7aF9w ++ rm /tmp/tmp.l5wnKc8vO0 /tmp/tmp.Dk1uz7aF9w ++ return 0 + [[ 3 == \3 ]] + compare_mysql_cmd select-3 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\''' + local command_id=select-3 + local 'command=SHOW DATABASES;' + local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-3-80.sql ']' + run_mysql 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\''' + local 'command=SHOW DATABASES;' + local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fOM2VL8Y6B +++ mktemp ++ local LAST_ERR=/tmp/tmp.Yia3AKa4tK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fOM2VL8Y6B ++ cat /tmp/tmp.Yia3AKa4tK ++ rm /tmp/tmp.fOM2VL8Y6B /tmp/tmp.Yia3AKa4tK ++ return 0 + client_pod=pxc-client-65d95667fc-5hskv + wait_pod pxc-client-65d95667fc-5hskv + local pod=pxc-client-65d95667fc-5hskv + local max_retry=480 + local ns= ++ echo pxc-client-65d95667fc-5hskv ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-65d95667fc-5hskv condition met pxc-client-65d95667fc-5hskv.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.TxLWXMXQ5Q/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1421/e2e-tests/users/compare/select-3.sql /tmp/tmp.TxLWXMXQ5Q/select-3.sql + destroy users-17620 + local namespace=users-17620 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v 'the object has been modified' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + grep -v 'get backup status: Job.batch' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v level=info + sort -u + tee /tmp/tmp.TxLWXMXQ5Q/operator.log +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.OI9EMk55dy +++ mktemp ++ local LAST_ERR=/tmp/tmp.IpHQDyim66 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OI9EMk55dy ++ cat /tmp/tmp.IpHQDyim66 ++ rm /tmp/tmp.OI9EMk55dy /tmp/tmp.IpHQDyim66 ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-5c699b45b5-s9pfn ++ mktemp + local LAST_OUT=/tmp/tmp.PSJq0cb3FH ++ mktemp + local LAST_ERR=/tmp/tmp.YFm31KbxBY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-5c699b45b5-s9pfn + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PSJq0cb3FH + cat /tmp/tmp.YFm31KbxBY + rm /tmp/tmp.PSJq0cb3FH /tmp/tmp.YFm31KbxBY + return 0 2023-12-19T10:11:30.592Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.25.16-gke.1041000"} 2023-12-19T10:11:30.593Z INFO setup Manager starting up {"gitCommit": "86089695f1e763ec273f5c9d9614a64c0857fed0", "gitBranch": "PR-1421-86089695", "buildTime": "2023-12-19T08:04:51Z", "goVersion": "go1.21.5", "os": "linux", "arch": "amd64"} 2023-12-19T10:11:30.593Z INFO setup Registering Components. 2023-12-19T10:11:32.409Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2023-12-19T10:11:32.409Z INFO controller-runtime.metrics Starting metrics server 2023-12-19T10:11:32.409Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2023-12-19T10:11:32.409Z INFO setup Starting the Cmd. 2023-12-19T10:11:32.492Z INFO attempting to acquire leader lease pxc-operator/08db0feb.percona.com... 2023-12-19T10:11:32.492Z INFO controller-runtime.webhook Starting webhook server 2023-12-19T10:11:32.492Z INFO starting server {"kind": "health probe", "addr": "[::]:8081"} 2023-12-19T10:11:32.493Z INFO controller-runtime.certwatcher Starting certificate watcher 2023-12-19T10:11:32.493Z INFO controller-runtime.certwatcher Updated current TLS certificate 2023-12-19T10:11:32.493Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2023-12-19T10:11:32.503Z DEBUG events percona-xtradb-cluster-operator-5c699b45b5-s9pfn_a250fae1-3b12-4f00-9bd3-1fcbef6034bd became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db0feb.percona.com","uid":"4fd96e71-02e2-4abc-b149-245c78bc3f29","apiVersion":"coordination.k8s.io/v1","resourceVersion":"63625"}, "reason": "LeaderElection"} 2023-12-19T10:11:32.503Z INFO successfully acquired lease pxc-operator/08db0feb.percona.com 2023-12-19T10:11:32.504Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2023-12-19T10:11:32.504Z INFO Starting Controller {"controller": "pxc-controller"} 2023-12-19T10:11:32.504Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2023-12-19T10:11:32.504Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2023-12-19T10:11:32.504Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"} 2023-12-19T10:11:32.504Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2023-12-19T10:11:32.712Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2023-12-19T10:11:32.712Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2023-12-19T10:11:32.712Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2023-12-19T10:11:58.543Z INFO Set CR version {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3c07f98d-0275-47a1-a20c-ad00a12794e8", "version": "1.14.0"} 2023-12-19T10:13:11.826Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ee95435c-cc9f-4032-9818-b28c9d64c565", "user": "operator"} 2023-12-19T10:13:11.866Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ee95435c-cc9f-4032-9818-b28c9d64c565", "user": "monitor"} 2023-12-19T10:13:11.964Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ee95435c-cc9f-4032-9818-b28c9d64c565"} 2023-12-19T10:13:12.007Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ee95435c-cc9f-4032-9818-b28c9d64c565"} 2023-12-19T10:13:12.043Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ee95435c-cc9f-4032-9818-b28c9d64c565", "user": "xtrabackup"} 2023-12-19T10:13:12.095Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ee95435c-cc9f-4032-9818-b28c9d64c565"} 2023-12-19T10:13:12.132Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ee95435c-cc9f-4032-9818-b28c9d64c565", "user": "replication"} 2023-12-19T10:13:12.202Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ee95435c-cc9f-4032-9818-b28c9d64c565", "err": "get primary pxc pod: not found"} 2023-12-19T10:13:16.835Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "1ddf59ab-1e11-42ec-9120-43b522859aec", "err": "get primary pxc pod: not found"} 2023-12-19T10:13:22.147Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "24ef6ea4-3bb3-4714-b72f-f4619119ca47", "err": "get primary pxc pod: not found"} 2023-12-19T10:13:27.405Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a97a80c1-b33b-4712-aa4c-163eeab61712", "err": "get primary pxc pod: not found"} 2023-12-19T10:13:32.607Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "74ebbb60-71f0-49e8-a2e2-3b2882af3069", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:13:37.778Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "85fa59b7-2dfa-4aed-bcb5-dc04dbc12e34", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:13:43.006Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b6c511fc-b54c-40c0-9a20-a964e354f72a", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:13:48.193Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "899bd7c9-d250-4b15-9887-115b8a9b733f", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:13:53.367Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "04584efd-7d01-4b6c-b78c-2753d08b07f3", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:13:58.595Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6af243ea-9f7c-46ce-bd3a-3cf858ce6d43", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:03.771Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "93d675fa-6dfd-4ea9-a74e-76bedbe19c5d", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:09.027Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "42dc285b-e677-418e-a69b-3bdfe761cb1e", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:14.226Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "dff32f97-50ce-4ecc-89d3-21d11789d3f2", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:19.403Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "33e036ad-b36b-4a8e-995a-2df945cafc7c", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:24.582Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "de6f2aa6-5d87-4e20-8b57-2a407007eefc", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:30.011Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "abeb690e-206f-4462-b686-9e87f574692f", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:30.027Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "abeb690e-206f-4462-b686-9e87f574692f", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:14:30.324Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "dea0107a-371d-4fd4-8a7f-ea9103a9a89e", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:14:30.336Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "dea0107a-371d-4fd4-8a7f-ea9103a9a89e", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:35.230Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "565e30b7-ffad-4eaa-98c0-31e28df68abc", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:14:35.241Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "565e30b7-ffad-4eaa-98c0-31e28df68abc", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:40.461Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6e1e1d82-4f78-4820-91cd-9951f3f9424e", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:40.477Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6e1e1d82-4f78-4820-91cd-9951f3f9424e", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:14:45.704Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4cfe9ae8-3fc6-4d0f-8007-5d9b62d3b3dc", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:14:45.714Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4cfe9ae8-3fc6-4d0f-8007-5d9b62d3b3dc", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:50.927Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "42709d59-a1ab-412c-9936-3963e78b25cb", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:50.993Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "42709d59-a1ab-412c-9936-3963e78b25cb", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:14:56.188Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6b1de568-62ed-4b57-bec8-1d16aa6fe410", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:14:56.202Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6b1de568-62ed-4b57-bec8-1d16aa6fe410", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:01.401Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "03a4fc9f-3162-47a5-940b-fecf08fe5387", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:01.413Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "03a4fc9f-3162-47a5-940b-fecf08fe5387", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:06.692Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ca196d16-54a1-4e6a-aff1-1df26d5bd8bf", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:06.703Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ca196d16-54a1-4e6a-aff1-1df26d5bd8bf", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:12.052Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c269c85e-5ec9-4a45-9756-95233457d65f", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:12.065Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c269c85e-5ec9-4a45-9756-95233457d65f", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:17.306Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f62c732f-e2f0-4058-94b5-b1daab5fdcd5", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:17.317Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f62c732f-e2f0-4058-94b5-b1daab5fdcd5", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:22.595Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "97d739f2-58d1-405f-8b8c-9f135f527df4", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:22.607Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "97d739f2-58d1-405f-8b8c-9f135f527df4", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:27.812Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7e581302-8aef-4864-8d76-b752d5272b25", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:27.825Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7e581302-8aef-4864-8d76-b752d5272b25", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:33.131Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "56ee7cf8-54da-437c-921d-11344973293d", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:33.142Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "56ee7cf8-54da-437c-921d-11344973293d", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:38.433Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "640a9add-e5eb-4a05-8a18-bf25c32c8f4c", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:38.496Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "640a9add-e5eb-4a05-8a18-bf25c32c8f4c", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:43.685Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3956fe46-5bbc-4b1a-9ec3-c1c395263eab", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:43.704Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3956fe46-5bbc-4b1a-9ec3-c1c395263eab", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:43.718Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3956fe46-5bbc-4b1a-9ec3-c1c395263eab", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:15:43.901Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c5bde0c3-cf07-481e-9848-257a9d70ba95", "user": "root"} 2023-12-19T10:15:44.104Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c5bde0c3-cf07-481e-9848-257a9d70ba95", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:44.114Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c5bde0c3-cf07-481e-9848-257a9d70ba95", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:15:44.123Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c5bde0c3-cf07-481e-9848-257a9d70ba95", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:44.137Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c5bde0c3-cf07-481e-9848-257a9d70ba95", "new version": "8.0.34-26.1"} 2023-12-19T10:15:45.302Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3390751e-e261-427d-ae5b-0c81883fac6b", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:45.312Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3390751e-e261-427d-ae5b-0c81883fac6b", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:45.321Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3390751e-e261-427d-ae5b-0c81883fac6b", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:15:46.106Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c5bde0c3-cf07-481e-9848-257a9d70ba95"} 2023-12-19T10:15:49.053Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2deb2a38-727e-4ecd-a234-ac999a047be7", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:49.101Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2deb2a38-727e-4ecd-a234-ac999a047be7", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:49.113Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2deb2a38-727e-4ecd-a234-ac999a047be7", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:15:51.020Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2deb2a38-727e-4ecd-a234-ac999a047be7"} 2023-12-19T10:15:54.391Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "07b03f08-8cb6-4a2e-9fcc-2f5894a95955", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:15:54.406Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "07b03f08-8cb6-4a2e-9fcc-2f5894a95955", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:54.418Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "07b03f08-8cb6-4a2e-9fcc-2f5894a95955", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:15:56.380Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "07b03f08-8cb6-4a2e-9fcc-2f5894a95955"} 2023-12-19T10:15:59.709Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "37d2260e-a37b-4f87-bd19-a2abcade67a0", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:15:59.724Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "37d2260e-a37b-4f87-bd19-a2abcade67a0", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:15:59.737Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "37d2260e-a37b-4f87-bd19-a2abcade67a0", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:16:01.772Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "37d2260e-a37b-4f87-bd19-a2abcade67a0"} 2023-12-19T10:16:05.029Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "34d2122a-5025-4b0b-aeab-a3ba4dd34c3b", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:16:05.042Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "34d2122a-5025-4b0b-aeab-a3ba4dd34c3b", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:16:05.053Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "34d2122a-5025-4b0b-aeab-a3ba4dd34c3b", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:16:07.023Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "34d2122a-5025-4b0b-aeab-a3ba4dd34c3b"} 2023-12-19T10:16:10.402Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f90fd3f9-544a-4799-932d-9c05f1d61c74", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:16:10.414Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f90fd3f9-544a-4799-932d-9c05f1d61c74", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:16:10.427Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f90fd3f9-544a-4799-932d-9c05f1d61c74", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:16:12.406Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f90fd3f9-544a-4799-932d-9c05f1d61c74"} 2023-12-19T10:16:15.672Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "fa6a0893-f9af-4092-87ff-50b363da5d84", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:16:15.695Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "fa6a0893-f9af-4092-87ff-50b363da5d84", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:16:15.705Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "fa6a0893-f9af-4092-87ff-50b363da5d84", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:16:17.820Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "fa6a0893-f9af-4092-87ff-50b363da5d84"} 2023-12-19T10:16:21.078Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ce608140-81f0-4fca-8408-b581c72978f0", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:16:21.089Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ce608140-81f0-4fca-8408-b581c72978f0", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:16:21.098Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ce608140-81f0-4fca-8408-b581c72978f0", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:16:23.217Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ce608140-81f0-4fca-8408-b581c72978f0"} 2023-12-19T10:16:26.401Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "00ea1695-9389-43a1-8de7-596abf14bbac", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:16:26.415Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "00ea1695-9389-43a1-8de7-596abf14bbac", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:16:26.424Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "00ea1695-9389-43a1-8de7-596abf14bbac", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:16:28.413Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "00ea1695-9389-43a1-8de7-596abf14bbac"} 2023-12-19T10:16:31.706Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "475bbb72-8ade-4308-9098-495f8d46776e", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:16:31.718Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "475bbb72-8ade-4308-9098-495f8d46776e", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:16:31.731Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "475bbb72-8ade-4308-9098-495f8d46776e", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:16:33.783Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "475bbb72-8ade-4308-9098-495f8d46776e"} 2023-12-19T10:16:37.197Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6ca3a922-a9cf-4ae3-8c4e-5c772847d1aa", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:16:37.213Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6ca3a922-a9cf-4ae3-8c4e-5c772847d1aa", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:16:37.221Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6ca3a922-a9cf-4ae3-8c4e-5c772847d1aa", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:16:39.286Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6ca3a922-a9cf-4ae3-8c4e-5c772847d1aa"} 2023-12-19T10:16:42.559Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "1c96d34d-a724-4395-88df-80f882fa62e7", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:16:42.573Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "1c96d34d-a724-4395-88df-80f882fa62e7", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:16:42.593Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "1c96d34d-a724-4395-88df-80f882fa62e7", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:16:44.687Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "1c96d34d-a724-4395-88df-80f882fa62e7"} 2023-12-19T10:16:47.689Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9ae5d050-f55d-45e3-9ec7-c453e8889bba", "user": "root"} 2023-12-19T10:16:47.740Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9ae5d050-f55d-45e3-9ec7-c453e8889bba", "user": "root"} 2023-12-19T10:16:50.570Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9ae5d050-f55d-45e3-9ec7-c453e8889bba"} 2023-12-19T10:16:50.577Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9ae5d050-f55d-45e3-9ec7-c453e8889bba", "user": "root"} 2023-12-19T10:16:50.629Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9ae5d050-f55d-45e3-9ec7-c453e8889bba", "user": "root"} 2023-12-19T10:16:50.821Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9ae5d050-f55d-45e3-9ec7-c453e8889bba", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:16:50.832Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9ae5d050-f55d-45e3-9ec7-c453e8889bba", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:16:50.842Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9ae5d050-f55d-45e3-9ec7-c453e8889bba", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:16:53.215Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9ae5d050-f55d-45e3-9ec7-c453e8889bba"} 2023-12-19T10:16:56.126Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "fddd1ae4-3417-4e95-9ceb-9e05c305bb0e", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:16:56.138Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "fddd1ae4-3417-4e95-9ceb-9e05c305bb0e", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:16:56.148Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "fddd1ae4-3417-4e95-9ceb-9e05c305bb0e", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:16:58.079Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "fddd1ae4-3417-4e95-9ceb-9e05c305bb0e"} 2023-12-19T10:17:01.423Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ae34c5ff-c126-4e77-8a1a-e3743e14aac3", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:01.435Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ae34c5ff-c126-4e77-8a1a-e3743e14aac3", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:01.445Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ae34c5ff-c126-4e77-8a1a-e3743e14aac3", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:02.543Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2c1c65bd-644f-4811-96b3-ed209b0c9d05", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:02.552Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2c1c65bd-644f-4811-96b3-ed209b0c9d05", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:02.565Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2c1c65bd-644f-4811-96b3-ed209b0c9d05", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:03.195Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "309121ed-a834-4d9b-8f0c-4e782df6dd73", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:03.210Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "309121ed-a834-4d9b-8f0c-4e782df6dd73", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:03.221Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "309121ed-a834-4d9b-8f0c-4e782df6dd73", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:03.575Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ae34c5ff-c126-4e77-8a1a-e3743e14aac3"} 2023-12-19T10:17:06.697Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0abbcaf8-06de-45ac-968d-eaee389bc4ec", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:06.708Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0abbcaf8-06de-45ac-968d-eaee389bc4ec", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:06.720Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0abbcaf8-06de-45ac-968d-eaee389bc4ec", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:12.194Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e88e4f6c-1d37-4f06-9626-1d306b3f0df0", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:12.205Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e88e4f6c-1d37-4f06-9626-1d306b3f0df0", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:12.217Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e88e4f6c-1d37-4f06-9626-1d306b3f0df0", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:17.410Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "337a0583-a1b7-4e8b-9823-d1cf582406dd", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:17.424Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "337a0583-a1b7-4e8b-9823-d1cf582406dd", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:17.436Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "337a0583-a1b7-4e8b-9823-d1cf582406dd", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:22.690Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "bc417232-6314-4003-9fc7-d48891229d8d", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:22.702Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "bc417232-6314-4003-9fc7-d48891229d8d", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:22.716Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "bc417232-6314-4003-9fc7-d48891229d8d", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:23.139Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c23fc3e6-ac64-4e62-b7d1-981509ecfc35", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:23.153Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c23fc3e6-ac64-4e62-b7d1-981509ecfc35", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:23.193Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c23fc3e6-ac64-4e62-b7d1-981509ecfc35", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:25.436Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c23fc3e6-ac64-4e62-b7d1-981509ecfc35", "error": "exec syncusers: command terminated with exit code 1 / / ERROR (line:387) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR (line:387) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:899\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1191\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"} 2023-12-19T10:17:28.062Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9988af86-c32d-4c99-be78-6cf3e62ade7a", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:28.094Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9988af86-c32d-4c99-be78-6cf3e62ade7a", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:28.105Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9988af86-c32d-4c99-be78-6cf3e62ade7a", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:30.445Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9988af86-c32d-4c99-be78-6cf3e62ade7a", "error": "exec syncusers: command terminated with exit code 1 / / ERROR (line:387) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR (line:387) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:899\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1191\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"} 2023-12-19T10:17:33.264Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3c8b9f16-5869-4ee5-bf12-181fb49818b0", "user": "proxyadmin"} 2023-12-19T10:17:33.264Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3c8b9f16-5869-4ee5-bf12-181fb49818b0", "user": "proxyadmin"} 2023-12-19T10:17:33.349Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3c8b9f16-5869-4ee5-bf12-181fb49818b0", "user": "proxyadmin"} 2023-12-19T10:17:33.357Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3c8b9f16-5869-4ee5-bf12-181fb49818b0", "user": "proxyadmin"} 2023-12-19T10:17:33.357Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3c8b9f16-5869-4ee5-bf12-181fb49818b0", "last-applied-secret": "1a55867d0f4fa0d0643653b2eb57b2f690315d8d843a10599c9839c2a779a95f"} 2023-12-19T10:17:33.493Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3c8b9f16-5869-4ee5-bf12-181fb49818b0", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:33.508Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3c8b9f16-5869-4ee5-bf12-181fb49818b0", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:33.521Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3c8b9f16-5869-4ee5-bf12-181fb49818b0", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:33.559Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3c8b9f16-5869-4ee5-bf12-181fb49818b0", "error": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:899\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1191\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"} 2023-12-19T10:17:33.818Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "fc049872-b0d3-466a-b58c-f0a1afd9b0f4", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:33.831Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "fc049872-b0d3-466a-b58c-f0a1afd9b0f4", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:33.841Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "fc049872-b0d3-466a-b58c-f0a1afd9b0f4", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:34.907Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "34dafd46-fea9-4cf3-9210-39d73690457a", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:34.921Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "34dafd46-fea9-4cf3-9210-39d73690457a", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:34.933Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "34dafd46-fea9-4cf3-9210-39d73690457a", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:36.575Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "827fe574-c994-4e99-a4ff-9b6930e95a5f", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:36.585Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "827fe574-c994-4e99-a4ff-9b6930e95a5f", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:36.596Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "827fe574-c994-4e99-a4ff-9b6930e95a5f", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:43.809Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "d5f5514f-e5c7-49fe-bfc2-bd31aac550d3", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:43.821Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "d5f5514f-e5c7-49fe-bfc2-bd31aac550d3", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:43.835Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "d5f5514f-e5c7-49fe-bfc2-bd31aac550d3", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:49.116Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b429105b-624c-40d4-8394-afa7ab51e137", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:49.128Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b429105b-624c-40d4-8394-afa7ab51e137", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:49.138Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b429105b-624c-40d4-8394-afa7ab51e137", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:49.331Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "105dcbbb-14d4-40c0-970b-1a7eb43eded5", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:49.343Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "105dcbbb-14d4-40c0-970b-1a7eb43eded5", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:49.354Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "105dcbbb-14d4-40c0-970b-1a7eb43eded5", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:54.355Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "80fed98d-fafd-4222-800c-f0a5f9a9128f", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:54.367Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "80fed98d-fafd-4222-800c-f0a5f9a9128f", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:54.393Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "80fed98d-fafd-4222-800c-f0a5f9a9128f", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:17:59.587Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "79e01070-d944-4d46-b54c-bb5ae6e78952", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:17:59.600Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "79e01070-d944-4d46-b54c-bb5ae6e78952", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:17:59.613Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "79e01070-d944-4d46-b54c-bb5ae6e78952", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:18:04.800Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "57d31dd3-aa8b-426d-89b9-4b7b5a2d91c8", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:18:04.812Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "57d31dd3-aa8b-426d-89b9-4b7b5a2d91c8", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:18:04.822Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "57d31dd3-aa8b-426d-89b9-4b7b5a2d91c8", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:18:14.998Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0d612a03-c437-43b1-be22-ccca94296b75", "err": "get primary pxc pod: not found"} 2023-12-19T10:18:20.250Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6f590266-1473-4d40-86da-0311dd3e7402", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:18:20.266Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6f590266-1473-4d40-86da-0311dd3e7402", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:18:20.295Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6f590266-1473-4d40-86da-0311dd3e7402", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:18:20.626Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e078bd8f-6274-414b-a314-c5f293437786", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:18:20.638Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e078bd8f-6274-414b-a314-c5f293437786", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:18:20.693Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e078bd8f-6274-414b-a314-c5f293437786", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:18:25.188Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e078bd8f-6274-414b-a314-c5f293437786"} 2023-12-19T10:18:25.605Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "83450d52-cf96-4de9-8106-aae2740103f2", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:18:25.615Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "83450d52-cf96-4de9-8106-aae2740103f2", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:18:25.628Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "83450d52-cf96-4de9-8106-aae2740103f2", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:18:28.638Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "83450d52-cf96-4de9-8106-aae2740103f2"} 2023-12-19T10:18:30.852Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "785e4e86-dea9-476a-bc5e-67e70b394a96", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:18:30.864Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "785e4e86-dea9-476a-bc5e-67e70b394a96", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:18:30.877Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "785e4e86-dea9-476a-bc5e-67e70b394a96", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:18:31.167Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4d94d0e5-c015-436d-8614-605e56d34d76", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:18:31.194Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4d94d0e5-c015-436d-8614-605e56d34d76", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:18:31.206Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4d94d0e5-c015-436d-8614-605e56d34d76", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:18:34.210Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4d94d0e5-c015-436d-8614-605e56d34d76"} 2023-12-19T10:18:36.205Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "68f8f0c1-b947-4217-99ab-d2688d1c4a11", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:18:36.218Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "68f8f0c1-b947-4217-99ab-d2688d1c4a11", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:18:36.233Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "68f8f0c1-b947-4217-99ab-d2688d1c4a11", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:18:39.218Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "68f8f0c1-b947-4217-99ab-d2688d1c4a11"} 2023-12-19T10:18:40.024Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7f8d100e-ba1a-4071-b2c7-7b5994396de5", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:18:40.036Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7f8d100e-ba1a-4071-b2c7-7b5994396de5", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:18:40.048Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7f8d100e-ba1a-4071-b2c7-7b5994396de5", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:18:40.379Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c773e7c3-71f6-46ed-aa1e-67f75d2ce862", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:18:40.396Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c773e7c3-71f6-46ed-aa1e-67f75d2ce862", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:18:40.407Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c773e7c3-71f6-46ed-aa1e-67f75d2ce862", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:18:41.993Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7f8d100e-ba1a-4071-b2c7-7b5994396de5"} 2023-12-19T10:18:46.640Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "719ecbcb-4c44-4470-8caf-f5ffe0850308", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:18:46.651Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "719ecbcb-4c44-4470-8caf-f5ffe0850308", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:18:46.663Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "719ecbcb-4c44-4470-8caf-f5ffe0850308", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:18:46.818Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "5b0fc250-8087-4a26-a279-2d9df8a40d75", "user": "xtrabackup"} 2023-12-19T10:18:46.852Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "5b0fc250-8087-4a26-a279-2d9df8a40d75", "user": "xtrabackup"} 2023-12-19T10:18:46.860Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "5b0fc250-8087-4a26-a279-2d9df8a40d75", "user": "xtrabackup"} 2023-12-19T10:18:46.894Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "5b0fc250-8087-4a26-a279-2d9df8a40d75", "user": "xtrabackup"} 2023-12-19T10:18:46.906Z INFO PXC pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "5b0fc250-8087-4a26-a279-2d9df8a40d75", "last-applied-secret": "d11bcc70ca53ea410cd455e13d82e7225917b0861d9b677ed34c02c5f6fbaf1b"} 2023-12-19T10:18:47.246Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "5b0fc250-8087-4a26-a279-2d9df8a40d75", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:18:47.261Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "5b0fc250-8087-4a26-a279-2d9df8a40d75", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:18:47.273Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "5b0fc250-8087-4a26-a279-2d9df8a40d75", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:18:50.277Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "5b0fc250-8087-4a26-a279-2d9df8a40d75"} 2023-12-19T10:19:39.345Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b6021f61-9f9f-4d97-950a-051a651315a5", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:19:39.387Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b6021f61-9f9f-4d97-950a-051a651315a5", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-17620 on 10.83.0.10:53: no such host"} 2023-12-19T10:19:44.151Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "86e39b31-4b0b-4d79-9be1-142dc3ac9216", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:19:44.163Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "86e39b31-4b0b-4d79-9be1-142dc3ac9216", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:19:44.208Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "86e39b31-4b0b-4d79-9be1-142dc3ac9216", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-17620 on 10.83.0.10:53: no such host"} 2023-12-19T10:19:49.440Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eabdadab-55e8-422d-b609-691d551cc4be", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:19:49.450Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eabdadab-55e8-422d-b609-691d551cc4be", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:19:49.453Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eabdadab-55e8-422d-b609-691d551cc4be", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp 10.82.250.36:33062: connect: connection refused"} 2023-12-19T10:19:54.647Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "df574379-aa7c-4a2c-a884-260ec5274892", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:19:54.674Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "df574379-aa7c-4a2c-a884-260ec5274892", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:19:59.850Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "73ec8a7d-b6b9-4a06-9343-c3e9f9931d1e", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:19:59.865Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "73ec8a7d-b6b9-4a06-9343-c3e9f9931d1e", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:20:05.048Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f089d7b6-4f4a-4ff2-9c7a-a60afeb5a034", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:20:05.060Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f089d7b6-4f4a-4ff2-9c7a-a60afeb5a034", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:20:10.260Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "69f52986-af6e-4351-abf0-65d7bf7c922d", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:20:10.277Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "69f52986-af6e-4351-abf0-65d7bf7c922d", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:20:15.466Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6a640f65-971f-4592-8ba6-e112ada713b1", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:20:15.492Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6a640f65-971f-4592-8ba6-e112ada713b1", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:20:20.727Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "54676a68-2813-402a-939a-37677c2189d6", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:20:20.742Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "54676a68-2813-402a-939a-37677c2189d6", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:20:25.910Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2e322f09-9d19-48c0-94be-c80dbabc6371", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:20:25.924Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2e322f09-9d19-48c0-94be-c80dbabc6371", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:20:31.130Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e3225b4a-e7ef-4710-b560-6e4053a4f366", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:20:31.143Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e3225b4a-e7ef-4710-b560-6e4053a4f366", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:20:31.154Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e3225b4a-e7ef-4710-b560-6e4053a4f366", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:20:31.401Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0d0ab830-7dd1-4271-a7fd-f0fd86104234", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:20:31.414Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0d0ab830-7dd1-4271-a7fd-f0fd86104234", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:20:31.425Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0d0ab830-7dd1-4271-a7fd-f0fd86104234", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:20:36.648Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3cf8e732-ab90-46c6-96d8-8795aa1f42a9", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:20:36.664Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3cf8e732-ab90-46c6-96d8-8795aa1f42a9", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:20:36.676Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3cf8e732-ab90-46c6-96d8-8795aa1f42a9", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:20:42.236Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b975ce6a-b7e1-42d1-9b61-96ea4abad8d0", "err": "failed to connect to pod some-name-pxc-0: dial tcp 10.82.249.51:33062: connect: connection refused"} 2023-12-19T10:20:47.505Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7b52789a-03cd-44b6-afb3-9805a5a04363", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:20:47.516Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7b52789a-03cd-44b6-afb3-9805a5a04363", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:20:52.669Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "49b3018b-c770-486a-91a9-57c658dc03bf", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:20:52.682Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "49b3018b-c770-486a-91a9-57c658dc03bf", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:20:57.840Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "14683ad5-e10f-4a25-ab42-9537a183ef9e", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:20:57.850Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "14683ad5-e10f-4a25-ab42-9537a183ef9e", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:03.038Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "33b1b9c7-36f4-41f2-a47b-9af169c268a5", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:03.050Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "33b1b9c7-36f4-41f2-a47b-9af169c268a5", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:08.253Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f33109c3-5c8d-45c2-b032-c60ff61fc602", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:08.269Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f33109c3-5c8d-45c2-b032-c60ff61fc602", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:13.417Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4262b645-6d03-4502-a038-a0a4828b1ef8", "primary name": "some-name-pxc-0.some-name-pxc.users-17620.svc.cluster.local"} 2023-12-19T10:21:18.642Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "47237493-1a11-472a-abe5-3f1c03661570", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:18.654Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "47237493-1a11-472a-abe5-3f1c03661570", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:21:18.667Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "47237493-1a11-472a-abe5-3f1c03661570", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:18.994Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ca5dea6d-b28e-4c38-bd7a-bc821eb59085", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:21:19.005Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ca5dea6d-b28e-4c38-bd7a-bc821eb59085", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:19.019Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ca5dea6d-b28e-4c38-bd7a-bc821eb59085", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:20.904Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ca5dea6d-b28e-4c38-bd7a-bc821eb59085"} 2023-12-19T10:21:23.993Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "88d453d0-20bb-47ff-bbbc-56b08f7d0b97", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:24.003Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "88d453d0-20bb-47ff-bbbc-56b08f7d0b97", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:24.016Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "88d453d0-20bb-47ff-bbbc-56b08f7d0b97", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:21:25.977Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "88d453d0-20bb-47ff-bbbc-56b08f7d0b97"} 2023-12-19T10:21:29.309Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b2c34412-8762-4b86-a9f8-6a4a690cf134", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:29.319Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b2c34412-8762-4b86-a9f8-6a4a690cf134", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:21:29.332Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b2c34412-8762-4b86-a9f8-6a4a690cf134", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:31.294Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b2c34412-8762-4b86-a9f8-6a4a690cf134"} 2023-12-19T10:21:34.634Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "07d016e6-cbb1-4500-b396-1b613038cd5b", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:34.648Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "07d016e6-cbb1-4500-b396-1b613038cd5b", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:34.658Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "07d016e6-cbb1-4500-b396-1b613038cd5b", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:21:36.573Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "07d016e6-cbb1-4500-b396-1b613038cd5b"} 2023-12-19T10:21:39.783Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "262f98ad-1600-4848-a66e-bfac7f096f99", "user": "monitor"} 2023-12-19T10:21:39.812Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "262f98ad-1600-4848-a66e-bfac7f096f99", "user": "monitor"} 2023-12-19T10:21:39.870Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "262f98ad-1600-4848-a66e-bfac7f096f99", "user": "monitor"} 2023-12-19T10:21:39.877Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "262f98ad-1600-4848-a66e-bfac7f096f99", "user": "monitor"} 2023-12-19T10:21:39.964Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "262f98ad-1600-4848-a66e-bfac7f096f99", "last-applied-secret": "4e3282050745046d5044e0fcb5f2bf140d7320a1256a79c8ca2184f38f4f3098"} 2023-12-19T10:21:40.101Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "262f98ad-1600-4848-a66e-bfac7f096f99", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:40.114Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "262f98ad-1600-4848-a66e-bfac7f096f99", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:40.129Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "262f98ad-1600-4848-a66e-bfac7f096f99", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:21:40.433Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "cd36130e-c079-4fb5-a845-84ad1656711f", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:40.442Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "cd36130e-c079-4fb5-a845-84ad1656711f", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:21:40.453Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "cd36130e-c079-4fb5-a845-84ad1656711f", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:41.519Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "262f98ad-1600-4848-a66e-bfac7f096f99", "error": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") / / ", "errorVerbose": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") / / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:899\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1191\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"} 2023-12-19T10:21:45.382Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f9059918-a231-498c-9666-86b66f406aed", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:21:45.402Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f9059918-a231-498c-9666-86b66f406aed", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:45.413Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f9059918-a231-498c-9666-86b66f406aed", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:45.640Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3d9cd59c-f18e-4e8c-91d3-f4e37fe7cf49", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:45.653Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3d9cd59c-f18e-4e8c-91d3-f4e37fe7cf49", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:45.664Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3d9cd59c-f18e-4e8c-91d3-f4e37fe7cf49", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:21:50.624Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c89cb6e6-8ca1-4127-966b-610707dbb4a5", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:21:50.636Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c89cb6e6-8ca1-4127-966b-610707dbb4a5", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:50.647Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c89cb6e6-8ca1-4127-966b-610707dbb4a5", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:21:55.911Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "673ecbd9-a861-44e1-a117-0402d3e71cb9", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:21:55.924Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "673ecbd9-a861-44e1-a117-0402d3e71cb9", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:21:55.935Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "673ecbd9-a861-44e1-a117-0402d3e71cb9", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:22:01.257Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "8c9e8609-420a-43cc-950b-f5b6103b5804", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:22:01.273Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "8c9e8609-420a-43cc-950b-f5b6103b5804", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:22:01.283Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "8c9e8609-420a-43cc-950b-f5b6103b5804", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:22:22.686Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a1c1d17c-391b-461c-a845-0b6103d3e456", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:22:22.700Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a1c1d17c-391b-461c-a845-0b6103d3e456", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:22:22.713Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a1c1d17c-391b-461c-a845-0b6103d3e456", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:22:27.946Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "222ae113-1bb1-4a9c-b372-8155dc9e8249", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:22:27.959Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "222ae113-1bb1-4a9c-b372-8155dc9e8249", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:22:27.969Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "222ae113-1bb1-4a9c-b372-8155dc9e8249", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:22:33.189Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e2a2d150-a67e-4a2f-8342-9ed0f6f99b30", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:22:33.202Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e2a2d150-a67e-4a2f-8342-9ed0f6f99b30", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:22:33.214Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e2a2d150-a67e-4a2f-8342-9ed0f6f99b30", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:22:33.356Z INFO Password updated but old one not discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e502808e-cf08-4158-a409-9d90dce15baa", "user": "monitor"} 2023-12-19T10:22:33.610Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e502808e-cf08-4158-a409-9d90dce15baa", "user": "monitor"} 2023-12-19T10:22:33.632Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e502808e-cf08-4158-a409-9d90dce15baa", "last-applied-secret": "4e3282050745046d5044e0fcb5f2bf140d7320a1256a79c8ca2184f38f4f3098"} 2023-12-19T10:22:33.736Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e502808e-cf08-4158-a409-9d90dce15baa", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:22:33.746Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e502808e-cf08-4158-a409-9d90dce15baa", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:22:33.756Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e502808e-cf08-4158-a409-9d90dce15baa", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:22:35.606Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e502808e-cf08-4158-a409-9d90dce15baa"} 2023-12-19T10:22:38.520Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4bb7f45c-6cbe-4ebc-b8df-f005af0caeb6", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:22:38.532Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4bb7f45c-6cbe-4ebc-b8df-f005af0caeb6", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:22:38.543Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4bb7f45c-6cbe-4ebc-b8df-f005af0caeb6", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:22:40.418Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4bb7f45c-6cbe-4ebc-b8df-f005af0caeb6"} 2023-12-19T10:22:43.848Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6b6e05c1-1b93-49f9-af34-a4b7fc9a9aaf", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:22:43.860Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6b6e05c1-1b93-49f9-af34-a4b7fc9a9aaf", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:22:43.871Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6b6e05c1-1b93-49f9-af34-a4b7fc9a9aaf", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:22:45.905Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6b6e05c1-1b93-49f9-af34-a4b7fc9a9aaf"} 2023-12-19T10:22:49.250Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e7f547b0-88b5-4e5b-b4bd-075bb59df821", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:22:49.263Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e7f547b0-88b5-4e5b-b4bd-075bb59df821", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:22:49.275Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e7f547b0-88b5-4e5b-b4bd-075bb59df821", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:22:51.328Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e7f547b0-88b5-4e5b-b4bd-075bb59df821"} 2023-12-19T10:22:54.620Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "8e2fdf30-7533-4306-bc80-e9fe7cb51e67", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:22:54.632Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "8e2fdf30-7533-4306-bc80-e9fe7cb51e67", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:22:54.643Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "8e2fdf30-7533-4306-bc80-e9fe7cb51e67", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:22:56.609Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "8e2fdf30-7533-4306-bc80-e9fe7cb51e67"} 2023-12-19T10:22:59.786Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e9cd4696-d3a3-4057-a7dc-7a613c9a5b2a", "user": "operator"} 2023-12-19T10:22:59.820Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e9cd4696-d3a3-4057-a7dc-7a613c9a5b2a", "user": "operator"} 2023-12-19T10:22:59.829Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e9cd4696-d3a3-4057-a7dc-7a613c9a5b2a", "user": "operator"} 2023-12-19T10:22:59.861Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e9cd4696-d3a3-4057-a7dc-7a613c9a5b2a", "user": "operator"} 2023-12-19T10:22:59.898Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e9cd4696-d3a3-4057-a7dc-7a613c9a5b2a", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"} 2023-12-19T10:23:00.065Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e9cd4696-d3a3-4057-a7dc-7a613c9a5b2a", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:23:00.078Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e9cd4696-d3a3-4057-a7dc-7a613c9a5b2a", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:23:00.094Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e9cd4696-d3a3-4057-a7dc-7a613c9a5b2a", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:23:00.438Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "1439cb36-1f4f-4a5a-b38b-aa0dd91f398d", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:23:00.448Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "1439cb36-1f4f-4a5a-b38b-aa0dd91f398d", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:23:00.460Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "1439cb36-1f4f-4a5a-b38b-aa0dd91f398d", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:23:00.679Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e9cd4696-d3a3-4057-a7dc-7a613c9a5b2a", "error": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-17620.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-17620.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-17620.svc.' (using password: YES)\nERROR (line:1437) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-17620.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-17620.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-17620.svc.' (using password: YES)\nERROR (line:1437) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:899\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1191\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"} 2023-12-19T10:23:05.436Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f93ed155-4beb-4c05-9812-9c6bcdf0de62", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:23:05.447Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f93ed155-4beb-4c05-9812-9c6bcdf0de62", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:23:05.458Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f93ed155-4beb-4c05-9812-9c6bcdf0de62", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:23:37.694Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "41d8ff58-0073-4bb0-b49a-c5cc021cc933", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:23:37.707Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "41d8ff58-0073-4bb0-b49a-c5cc021cc933", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:23:37.718Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "41d8ff58-0073-4bb0-b49a-c5cc021cc933", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:23:38.046Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ea349423-f1f3-4b77-9135-c87d2bc31800", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:23:38.058Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ea349423-f1f3-4b77-9135-c87d2bc31800", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:23:38.073Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ea349423-f1f3-4b77-9135-c87d2bc31800", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:23:40.874Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ea349423-f1f3-4b77-9135-c87d2bc31800"} 2023-12-19T10:23:43.041Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3b9eaaa1-814b-4222-8f57-75e5d83e56e1", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:23:43.052Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3b9eaaa1-814b-4222-8f57-75e5d83e56e1", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:23:43.096Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3b9eaaa1-814b-4222-8f57-75e5d83e56e1", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:23:44.925Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3b9eaaa1-814b-4222-8f57-75e5d83e56e1"} 2023-12-19T10:23:48.419Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b05289b7-8267-411f-ba0b-f0a389898f66", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:23:48.431Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b05289b7-8267-411f-ba0b-f0a389898f66", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:23:48.442Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b05289b7-8267-411f-ba0b-f0a389898f66", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:23:50.328Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b05289b7-8267-411f-ba0b-f0a389898f66"} 2023-12-19T10:23:53.706Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "89c7ab47-b826-4dbd-b6c8-ccae222cccbe", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:23:53.718Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "89c7ab47-b826-4dbd-b6c8-ccae222cccbe", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:23:53.728Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "89c7ab47-b826-4dbd-b6c8-ccae222cccbe", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:23:55.614Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "89c7ab47-b826-4dbd-b6c8-ccae222cccbe"} 2023-12-19T10:23:59.005Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9e92c293-dfdf-44a1-9f58-098c851c1a6a", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:23:59.018Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9e92c293-dfdf-44a1-9f58-098c851c1a6a", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:23:59.029Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9e92c293-dfdf-44a1-9f58-098c851c1a6a", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:24:00.393Z INFO Created user secrets {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "secrets": "my-cluster-secrets-2"} 2023-12-19T10:24:00.403Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "root"} 2023-12-19T10:24:00.452Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "root"} 2023-12-19T10:24:01.417Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9e92c293-dfdf-44a1-9f58-098c851c1a6a"} 2023-12-19T10:24:03.144Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd"} 2023-12-19T10:24:03.151Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "root"} 2023-12-19T10:24:03.203Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "root"} 2023-12-19T10:24:03.216Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "operator"} 2023-12-19T10:24:03.245Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "operator"} 2023-12-19T10:24:03.252Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "operator"} 2023-12-19T10:24:03.288Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "operator"} 2023-12-19T10:24:03.299Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "monitor"} 2023-12-19T10:24:03.331Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "monitor"} 2023-12-19T10:24:03.375Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "monitor"} 2023-12-19T10:24:03.384Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "monitor"} 2023-12-19T10:24:03.483Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "xtrabackup"} 2023-12-19T10:24:03.514Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "xtrabackup"} 2023-12-19T10:24:03.522Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "xtrabackup"} 2023-12-19T10:24:03.551Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "xtrabackup"} 2023-12-19T10:24:03.562Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "replication"} 2023-12-19T10:24:03.591Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "replication"} 2023-12-19T10:24:03.601Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "replication"} 2023-12-19T10:24:03.634Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "replication"} 2023-12-19T10:24:03.634Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "proxyadmin"} 2023-12-19T10:24:03.684Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "proxyadmin"} 2023-12-19T10:24:03.692Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "user": "proxyadmin"} 2023-12-19T10:24:03.692Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "last-applied-secret": "18869acc531cdb9f26c2c8dc013ab94c09b126c207253e8758594ee4b44fb44a"} 2023-12-19T10:24:03.692Z INFO PXC pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "last-applied-secret": "18869acc531cdb9f26c2c8dc013ab94c09b126c207253e8758594ee4b44fb44a"} 2023-12-19T10:24:03.866Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:24:03.903Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:24:03.921Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:24:04.001Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "eddcd71c-0f34-4b7b-91d4-255abda865cd", "error": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:899\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1191\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"} 2023-12-19T10:24:56.378Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "54709bd2-dc75-44e0-be88-922383667989", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:24:56.418Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "54709bd2-dc75-44e0-be88-922383667989", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-17620 on 10.83.0.10:53: no such host"} 2023-12-19T10:25:01.326Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e87c5813-197f-4420-995b-708d39a549d9", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-17620 on 10.83.0.10:53: no such host"} 2023-12-19T10:25:06.699Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "41a15078-0951-4619-ac03-356120e1e039", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:25:06.714Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "41a15078-0951-4619-ac03-356120e1e039", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-17620 on 10.83.0.10:53: no such host"} 2023-12-19T10:25:12.133Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "8ad7e8d8-b6fd-4498-b5df-45d685e2b659", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:25:12.143Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "8ad7e8d8-b6fd-4498-b5df-45d685e2b659", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:25:17.323Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e65c01fb-0298-4f91-b919-7207d2b5fe46", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:25:17.333Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e65c01fb-0298-4f91-b919-7207d2b5fe46", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:25:22.546Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "d11f5de7-8066-408b-b02e-5b325bce7731", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:25:22.558Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "d11f5de7-8066-408b-b02e-5b325bce7731", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:25:27.753Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "d6341f6e-8f7a-4670-aaa3-8c2f29ecfbcc", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:25:27.765Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "d6341f6e-8f7a-4670-aaa3-8c2f29ecfbcc", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:25:32.970Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "bd703876-9ddb-409d-a638-c810d28379d5", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:25:32.979Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "bd703876-9ddb-409d-a638-c810d28379d5", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:25:38.151Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "55f34c5b-5dbb-4e83-ba71-29443a643b9a", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:25:38.163Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "55f34c5b-5dbb-4e83-ba71-29443a643b9a", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:25:43.393Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "66577ff7-c8b8-4c4f-b8d6-9a4654d5eadb", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:25:43.409Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "66577ff7-c8b8-4c4f-b8d6-9a4654d5eadb", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:25:43.420Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "66577ff7-c8b8-4c4f-b8d6-9a4654d5eadb", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:25:43.616Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a1bb5087-37ea-4dd6-9421-617c3592e421", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:25:43.627Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a1bb5087-37ea-4dd6-9421-617c3592e421", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:25:43.639Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a1bb5087-37ea-4dd6-9421-617c3592e421", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:25:48.654Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b870172b-af9b-43f1-8123-0b1870ea3a87", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:25:48.667Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b870172b-af9b-43f1-8123-0b1870ea3a87", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:25:48.693Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b870172b-af9b-43f1-8123-0b1870ea3a87", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:25:53.859Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "cc9f19ca-6c8e-4438-811b-b7cd4e8f62d1", "err": "failed to connect to pod some-name-pxc-0: dial tcp 10.82.249.54:33062: connect: connection refused"} 2023-12-19T10:25:59.013Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0d83c593-c4c4-4b14-852e-6e1b7dd88fc8", "primary name": "some-name-pxc-0.some-name-pxc.users-17620.svc.cluster.local"} 2023-12-19T10:26:04.144Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "71f56d4c-c392-4f95-9594-fd3d137cb027", "primary name": "some-name-pxc-0.some-name-pxc.users-17620.svc.cluster.local"} 2023-12-19T10:26:09.304Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6f1ce3d9-88f7-457d-b516-db929cd5671a", "primary name": "some-name-pxc-0.some-name-pxc.users-17620.svc.cluster.local"} 2023-12-19T10:26:14.464Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "1a1ef733-287d-4b80-8bb4-5cebef29a96d", "primary name": "some-name-pxc-0.some-name-pxc.users-17620.svc.cluster.local"} 2023-12-19T10:26:19.586Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "d08d520c-d4d2-4070-b167-6127495152aa", "primary name": "some-name-pxc-0.some-name-pxc.users-17620.svc.cluster.local"} 2023-12-19T10:26:24.719Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e779b61e-3dc4-421f-a45b-220b30119f72", "primary name": "some-name-pxc-0.some-name-pxc.users-17620.svc.cluster.local"} 2023-12-19T10:26:29.933Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "326b116b-9815-455d-9a3f-fc8008ec6512", "primary name": "some-name-pxc-0.some-name-pxc.users-17620.svc.cluster.local"} 2023-12-19T10:26:35.131Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "404ec78b-d6ba-421c-b797-32f315f8b2f7", "primary name": "some-name-pxc-0.some-name-pxc.users-17620.svc.cluster.local"} 2023-12-19T10:26:40.330Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7c0a8abf-8dad-4e30-b84c-fd1416c1d081", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:26:40.343Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7c0a8abf-8dad-4e30-b84c-fd1416c1d081", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:26:40.393Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7c0a8abf-8dad-4e30-b84c-fd1416c1d081", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:26:40.526Z INFO Password updated but old one not discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "112e8840-8fe3-41e2-922e-b74d7f63617e", "user": "monitor"} 2023-12-19T10:26:40.782Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "112e8840-8fe3-41e2-922e-b74d7f63617e", "user": "monitor"} 2023-12-19T10:26:40.807Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "112e8840-8fe3-41e2-922e-b74d7f63617e", "last-applied-secret": "18869acc531cdb9f26c2c8dc013ab94c09b126c207253e8758594ee4b44fb44a"} 2023-12-19T10:26:40.960Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "112e8840-8fe3-41e2-922e-b74d7f63617e", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:26:40.972Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "112e8840-8fe3-41e2-922e-b74d7f63617e", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:26:40.996Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "112e8840-8fe3-41e2-922e-b74d7f63617e", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:26:42.814Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "112e8840-8fe3-41e2-922e-b74d7f63617e"} 2023-12-19T10:26:45.728Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ff843fe4-505a-4a15-8127-1f4c4dc93304", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:26:45.739Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ff843fe4-505a-4a15-8127-1f4c4dc93304", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:26:45.751Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ff843fe4-505a-4a15-8127-1f4c4dc93304", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:26:47.738Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ff843fe4-505a-4a15-8127-1f4c4dc93304"} 2023-12-19T10:26:50.887Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "24b33598-bb57-423d-99c6-a454ef5cc630", "user": "operator"} 2023-12-19T10:26:50.920Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "24b33598-bb57-423d-99c6-a454ef5cc630", "user": "operator"} 2023-12-19T10:26:50.928Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "24b33598-bb57-423d-99c6-a454ef5cc630", "user": "operator"} 2023-12-19T10:26:50.956Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "24b33598-bb57-423d-99c6-a454ef5cc630", "user": "operator"} 2023-12-19T10:26:50.990Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "24b33598-bb57-423d-99c6-a454ef5cc630", "last-applied-secret": "bf951418fc50db4b88d1b2fc389ba5cd79aa5bbf645ca85e6894fb131734b6ec"} 2023-12-19T10:26:51.147Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "24b33598-bb57-423d-99c6-a454ef5cc630", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:26:51.196Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "24b33598-bb57-423d-99c6-a454ef5cc630", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:26:51.206Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "24b33598-bb57-423d-99c6-a454ef5cc630", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:26:51.774Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "24b33598-bb57-423d-99c6-a454ef5cc630", "error": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-17620.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-17620.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-17620.svc.' (using password: YES)\nERROR (line:1437) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-17620.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-17620.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-17620.svc.' (using password: YES)\nERROR (line:1437) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:899\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1191\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"} 2023-12-19T10:26:59.684Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c3d740ad-f193-480d-9dc9-748a3163575a", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:26:59.695Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c3d740ad-f193-480d-9dc9-748a3163575a", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:26:59.707Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c3d740ad-f193-480d-9dc9-748a3163575a", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:26:59.936Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2be27342-023b-4ed1-a4b8-dde176462d12", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:26:59.997Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2be27342-023b-4ed1-a4b8-dde176462d12", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:27:00.012Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2be27342-023b-4ed1-a4b8-dde176462d12", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:27:00.696Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "75d0a03a-891a-49a7-b62a-67626d3909cd", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:27:00.709Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "75d0a03a-891a-49a7-b62a-67626d3909cd", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:27:00.721Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "75d0a03a-891a-49a7-b62a-67626d3909cd", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:27:34.020Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "edcb986f-958f-45d0-bd69-967a67f1dc29", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:27:34.035Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "edcb986f-958f-45d0-bd69-967a67f1dc29", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:27:34.047Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "edcb986f-958f-45d0-bd69-967a67f1dc29", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:27:34.550Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e532db75-2d32-4ba1-831e-7176c89a5e6d", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:27:34.593Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e532db75-2d32-4ba1-831e-7176c89a5e6d", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:27:34.604Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e532db75-2d32-4ba1-831e-7176c89a5e6d", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:27:37.739Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e532db75-2d32-4ba1-831e-7176c89a5e6d"} 2023-12-19T10:27:39.519Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2b54a236-b1e2-4caf-8ed6-6b8873912954", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:27:39.534Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2b54a236-b1e2-4caf-8ed6-6b8873912954", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:27:39.550Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2b54a236-b1e2-4caf-8ed6-6b8873912954", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:27:41.406Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2b54a236-b1e2-4caf-8ed6-6b8873912954"} 2023-12-19T10:27:44.827Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2acf724b-95ba-456b-9ac0-76325a09edc9", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:27:44.840Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2acf724b-95ba-456b-9ac0-76325a09edc9", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:27:44.892Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2acf724b-95ba-456b-9ac0-76325a09edc9", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:27:46.708Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2acf724b-95ba-456b-9ac0-76325a09edc9"} 2023-12-19T10:27:50.186Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "102063cd-031e-4f99-80c8-0dfd2821aaf3", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:27:50.199Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "102063cd-031e-4f99-80c8-0dfd2821aaf3", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:27:50.207Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "102063cd-031e-4f99-80c8-0dfd2821aaf3", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:27:52.229Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "102063cd-031e-4f99-80c8-0dfd2821aaf3"} 2023-12-19T10:27:55.451Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "47ba4d74-5bbb-48ae-b347-132c6095f7e8", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:27:55.465Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "47ba4d74-5bbb-48ae-b347-132c6095f7e8", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:27:55.492Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "47ba4d74-5bbb-48ae-b347-132c6095f7e8", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:27:57.317Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "47ba4d74-5bbb-48ae-b347-132c6095f7e8"} 2023-12-19T10:28:00.761Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9c9585f8-7e04-49a9-b8cf-1f0eb6ce06a4", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:28:00.774Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9c9585f8-7e04-49a9-b8cf-1f0eb6ce06a4", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:28:00.793Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9c9585f8-7e04-49a9-b8cf-1f0eb6ce06a4", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:28:02.644Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9c9585f8-7e04-49a9-b8cf-1f0eb6ce06a4"} 2023-12-19T10:28:06.062Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "cbaa82c0-d9a0-4756-a414-ef558c6b1501", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:28:06.075Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "cbaa82c0-d9a0-4756-a414-ef558c6b1501", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:28:06.093Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "cbaa82c0-d9a0-4756-a414-ef558c6b1501", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:28:08.050Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "cbaa82c0-d9a0-4756-a414-ef558c6b1501"} 2023-12-19T10:28:11.474Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2e644273-376f-4d8f-9b73-e1d8b7005f75", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:28:11.493Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2e644273-376f-4d8f-9b73-e1d8b7005f75", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:28:11.504Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2e644273-376f-4d8f-9b73-e1d8b7005f75", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:28:13.344Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2e644273-376f-4d8f-9b73-e1d8b7005f75"} 2023-12-19T10:28:16.765Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4b2ce34e-eae7-4376-8b9a-2a6df0b1b559", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:28:16.793Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4b2ce34e-eae7-4376-8b9a-2a6df0b1b559", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:28:16.806Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4b2ce34e-eae7-4376-8b9a-2a6df0b1b559", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:28:18.830Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4b2ce34e-eae7-4376-8b9a-2a6df0b1b559"} 2023-12-19T10:28:22.192Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9209ca24-de05-4071-825a-02968c694eaf", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:28:22.204Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9209ca24-de05-4071-825a-02968c694eaf", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:28:22.216Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9209ca24-de05-4071-825a-02968c694eaf", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:28:24.579Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9209ca24-de05-4071-825a-02968c694eaf"} 2023-12-19T10:28:27.663Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9edef9d6-4daf-4e4b-be22-39892166c7c1", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:28:27.675Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9edef9d6-4daf-4e4b-be22-39892166c7c1", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:28:27.686Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9edef9d6-4daf-4e4b-be22-39892166c7c1", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:28:30.212Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9edef9d6-4daf-4e4b-be22-39892166c7c1"} 2023-12-19T10:28:33.063Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a361305c-5f49-4a0a-a7c7-8fc1ac78afc7", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:28:33.075Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a361305c-5f49-4a0a-a7c7-8fc1ac78afc7", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:28:33.092Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a361305c-5f49-4a0a-a7c7-8fc1ac78afc7", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:28:35.198Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a361305c-5f49-4a0a-a7c7-8fc1ac78afc7"} 2023-12-19T10:28:38.408Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "de9f7b2a-245e-4d78-a215-e17ea968e0c7", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:28:38.422Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "de9f7b2a-245e-4d78-a215-e17ea968e0c7", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:28:38.436Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "de9f7b2a-245e-4d78-a215-e17ea968e0c7", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:28:40.434Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "de9f7b2a-245e-4d78-a215-e17ea968e0c7"} 2023-12-19T10:28:43.719Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "db843ef6-4f10-4aed-a704-192062c7f9c7", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:28:43.731Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "db843ef6-4f10-4aed-a704-192062c7f9c7", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:28:43.745Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "db843ef6-4f10-4aed-a704-192062c7f9c7", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:28:45.802Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "db843ef6-4f10-4aed-a704-192062c7f9c7"} 2023-12-19T10:28:49.048Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a05d2550-52f7-47e9-9105-eaabc89b92c3", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:28:49.061Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a05d2550-52f7-47e9-9105-eaabc89b92c3", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:28:49.072Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a05d2550-52f7-47e9-9105-eaabc89b92c3", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:28:51.043Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a05d2550-52f7-47e9-9105-eaabc89b92c3"} 2023-12-19T10:28:54.348Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "61fbd729-f53d-4a2f-84d9-4ab3f6278737", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:28:54.356Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "61fbd729-f53d-4a2f-84d9-4ab3f6278737", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:28:54.368Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "61fbd729-f53d-4a2f-84d9-4ab3f6278737", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:28:56.351Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "61fbd729-f53d-4a2f-84d9-4ab3f6278737"} 2023-12-19T10:28:59.646Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "af4e797e-4ce1-4f89-99f1-b1c4eff0af3d", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:28:59.657Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "af4e797e-4ce1-4f89-99f1-b1c4eff0af3d", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:28:59.692Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "af4e797e-4ce1-4f89-99f1-b1c4eff0af3d", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:29:01.710Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "af4e797e-4ce1-4f89-99f1-b1c4eff0af3d"} 2023-12-19T10:29:04.947Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ecc390b6-6064-4362-b56e-2f3ac02c3c97", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:29:04.955Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ecc390b6-6064-4362-b56e-2f3ac02c3c97", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:29:04.994Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ecc390b6-6064-4362-b56e-2f3ac02c3c97", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:29:06.925Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ecc390b6-6064-4362-b56e-2f3ac02c3c97"} 2023-12-19T10:29:10.093Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "root"} 2023-12-19T10:29:10.148Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "root"} 2023-12-19T10:29:13.251Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc"} 2023-12-19T10:29:13.260Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "root"} 2023-12-19T10:29:13.313Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "root"} 2023-12-19T10:29:13.335Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "monitor"} 2023-12-19T10:29:13.365Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "monitor"} 2023-12-19T10:29:13.411Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "monitor"} 2023-12-19T10:29:13.420Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "monitor"} 2023-12-19T10:29:13.502Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "xtrabackup"} 2023-12-19T10:29:13.529Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "xtrabackup"} 2023-12-19T10:29:13.536Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "xtrabackup"} 2023-12-19T10:29:13.566Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "xtrabackup"} 2023-12-19T10:29:13.577Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "proxyadmin"} 2023-12-19T10:29:13.625Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "proxyadmin"} 2023-12-19T10:29:13.634Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "user": "proxyadmin"} 2023-12-19T10:29:13.634Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "last-applied-secret": "99df19fc1513acad1be9ff958e910ced8a3e9be8f70f885c79094d12ffde760b"} 2023-12-19T10:29:13.634Z INFO PXC pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "last-applied-secret": "99df19fc1513acad1be9ff958e910ced8a3e9be8f70f885c79094d12ffde760b"} 2023-12-19T10:29:13.802Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:29:13.815Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:29:13.823Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "error": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:899\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1191\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1650"} 2023-12-19T10:29:13.827Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "758664e3-afa4-454f-a1c1-c55355b3aabc", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:31:05.430Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "1de8363d-e2d7-41ec-8a24-b7a679993369", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:31:05.474Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "1de8363d-e2d7-41ec-8a24-b7a679993369", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-17620 on 10.83.0.10:53: no such host"} 2023-12-19T10:31:10.426Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0d8de538-a3d3-485a-9508-6b2468ffc3cb", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:31:10.481Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0d8de538-a3d3-485a-9508-6b2468ffc3cb", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-17620 on 10.83.0.10:53: no such host"} 2023-12-19T10:31:15.664Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "58cff455-06f3-497a-b841-8dca68e4db15", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:31:15.674Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "58cff455-06f3-497a-b841-8dca68e4db15", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:31:20.881Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "bd4b4c7d-bb41-44be-b362-c7ed82863d78", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:31:20.893Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "bd4b4c7d-bb41-44be-b362-c7ed82863d78", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:31:26.053Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "714a47a4-45a6-4f69-874f-acc0f4cd7a10", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:31:26.101Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "714a47a4-45a6-4f69-874f-acc0f4cd7a10", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:31:31.258Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9341ff89-4a16-4274-919a-8a1dd608e1b8", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:31:31.268Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9341ff89-4a16-4274-919a-8a1dd608e1b8", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:31:36.601Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "be231df7-ba87-4907-ab94-ea77e95248c7", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:31:36.613Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "be231df7-ba87-4907-ab94-ea77e95248c7", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:31:41.804Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6952c853-f573-4b0b-b88f-3d9e3af15bbe", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:31:41.814Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6952c853-f573-4b0b-b88f-3d9e3af15bbe", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:31:46.990Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4aeb7cd5-eaae-4ec8-a46b-3751eafd1ce3", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:31:46.999Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4aeb7cd5-eaae-4ec8-a46b-3751eafd1ce3", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:31:52.215Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0171e6a4-56bf-41fe-a6b9-7518ef6581f5", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:31:52.228Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0171e6a4-56bf-41fe-a6b9-7518ef6581f5", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:31:57.417Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2f6764f2-b48f-4cec-ac1f-22e505b9bde1", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:31:57.428Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2f6764f2-b48f-4cec-ac1f-22e505b9bde1", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:31:57.492Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2f6764f2-b48f-4cec-ac1f-22e505b9bde1", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:31:57.710Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6cd90350-e317-4c1f-aef4-15c50583b76e", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:31:57.720Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6cd90350-e317-4c1f-aef4-15c50583b76e", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:31:57.732Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6cd90350-e317-4c1f-aef4-15c50583b76e", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:32:02.715Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "91e10737-86c6-46f6-87d7-455d950dc79b", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:32:02.726Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "91e10737-86c6-46f6-87d7-455d950dc79b", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:32:02.741Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "91e10737-86c6-46f6-87d7-455d950dc79b", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:32:07.956Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2df74caa-27d4-47bd-95d5-5fedf8cf0212", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:32:07.967Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2df74caa-27d4-47bd-95d5-5fedf8cf0212", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:32:13.135Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "d560e40b-fd28-47ba-a436-255ba5039d24", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:32:13.153Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "d560e40b-fd28-47ba-a436-255ba5039d24", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:32:18.322Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7374eda9-aa63-4814-ada6-857b82c7f016", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:32:18.341Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "7374eda9-aa63-4814-ada6-857b82c7f016", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:32:23.524Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c2436a69-2a8c-4d2b-91c9-d76db982655e", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:32:23.537Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c2436a69-2a8c-4d2b-91c9-d76db982655e", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:32:28.700Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "5fc326ce-7864-4418-bbf6-2a14133876ff", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:32:28.713Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "5fc326ce-7864-4418-bbf6-2a14133876ff", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:32:33.899Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3b7ce2fa-f602-465a-b163-1bf31440e1a0", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:32:33.911Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "3b7ce2fa-f602-465a-b163-1bf31440e1a0", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:32:39.393Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2575e035-06f3-43a3-b034-917114cf9ad5", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:32:39.406Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2575e035-06f3-43a3-b034-917114cf9ad5", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:32:44.594Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "bd63ec2a-04a8-4c0f-bba9-28cf8a6e3ca9", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:32:44.604Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "bd63ec2a-04a8-4c0f-bba9-28cf8a6e3ca9", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:32:44.616Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "bd63ec2a-04a8-4c0f-bba9-28cf8a6e3ca9", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:32:44.728Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "root"} 2023-12-19T10:32:44.781Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "root"} 2023-12-19T10:32:44.801Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "root"} 2023-12-19T10:32:44.851Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "root"} 2023-12-19T10:32:44.861Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "operator"} 2023-12-19T10:32:44.896Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "operator"} 2023-12-19T10:32:44.905Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "operator"} 2023-12-19T10:32:44.935Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "operator"} 2023-12-19T10:32:44.948Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "monitor"} 2023-12-19T10:32:44.977Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "monitor"} 2023-12-19T10:32:44.985Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "monitor"} 2023-12-19T10:32:45.104Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "xtrabackup"} 2023-12-19T10:32:45.135Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "xtrabackup"} 2023-12-19T10:32:45.143Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "xtrabackup"} 2023-12-19T10:32:45.174Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "xtrabackup"} 2023-12-19T10:32:45.185Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "replication"} 2023-12-19T10:32:45.215Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "replication"} 2023-12-19T10:32:45.223Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "replication"} 2023-12-19T10:32:45.252Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "user": "replication"} 2023-12-19T10:32:45.252Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"} 2023-12-19T10:32:45.252Z INFO PXC pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"} 2023-12-19T10:32:45.474Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:32:45.501Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:32:45.512Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4572168e-35f8-4347-be85-25b896c49ef6", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:33:37.238Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a73880c7-c49f-4eb9-9004-b64a901e09f0", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:33:37.252Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a73880c7-c49f-4eb9-9004-b64a901e09f0", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:33:37.267Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a73880c7-c49f-4eb9-9004-b64a901e09f0", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:33:58.072Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "21c29f0d-e8d7-483e-9714-c866ddf8f343", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:33:58.111Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "21c29f0d-e8d7-483e-9714-c866ddf8f343", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-17620 on 10.83.0.10:53: no such host"} 2023-12-19T10:34:03.285Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "36375be4-ab3a-4840-a62e-7699eafc27d8", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:34:03.296Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "36375be4-ab3a-4840-a62e-7699eafc27d8", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:34:03.512Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "185d7266-c747-4363-b9b2-08c89667b9e8", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:34:03.523Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "185d7266-c747-4363-b9b2-08c89667b9e8", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:34:08.596Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2bfd48dc-d0f1-404b-be0f-bb41a9f4b6fd", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:34:08.608Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2bfd48dc-d0f1-404b-be0f-bb41a9f4b6fd", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:34:13.767Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "30da5b8c-b1b8-4b4b-87a8-9e3fa36295f8", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:34:13.781Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "30da5b8c-b1b8-4b4b-87a8-9e3fa36295f8", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:34:18.946Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6f00542d-01bd-44b2-b2d8-2215fb2ddfab", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:34:18.958Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6f00542d-01bd-44b2-b2d8-2215fb2ddfab", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:34:24.122Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "edad3318-f31d-4cb2-a7b5-23e367f96fb7", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:34:24.131Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "edad3318-f31d-4cb2-a7b5-23e367f96fb7", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:34:29.320Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6632094d-3379-4e7f-a9b7-43ad8a064c0c", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:34:29.330Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6632094d-3379-4e7f-a9b7-43ad8a064c0c", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:34:29.342Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6632094d-3379-4e7f-a9b7-43ad8a064c0c", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:34:29.618Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "26133bd7-d49e-4cc2-a483-1d725a79d40e", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-17620 on 10.83.0.10:53: no such host"} 2023-12-19T10:34:34.571Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "9cb635b0-8c1c-41b5-b468-1c7a153ef8bd", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-17620 on 10.83.0.10:53: no such host"} 2023-12-19T10:34:54.756Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c8584417-694e-49c5-b7a7-95e3ca48de63", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:34:54.765Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c8584417-694e-49c5-b7a7-95e3ca48de63", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:34:59.975Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c4e0d697-5983-40b7-bfbd-df15f2c73638", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:34:59.988Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c4e0d697-5983-40b7-bfbd-df15f2c73638", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:05.156Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0a8ede8b-8602-4c62-8c48-1acf5c3ee69d", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:05.167Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "0a8ede8b-8602-4c62-8c48-1acf5c3ee69d", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:10.314Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2303d73b-47f0-4162-9ca7-9c61956dff67", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:10.327Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2303d73b-47f0-4162-9ca7-9c61956dff67", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:15.662Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c5d6e488-bdee-406b-a8c2-963751df1c8b", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:15.674Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c5d6e488-bdee-406b-a8c2-963751df1c8b", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:15.684Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c5d6e488-bdee-406b-a8c2-963751df1c8b", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:35:15.819Z INFO Password updated but old one not discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2829c89d-91a6-44e0-bd25-417e2c8a5cbd", "user": "monitor"} 2023-12-19T10:35:16.127Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2829c89d-91a6-44e0-bd25-417e2c8a5cbd", "user": "monitor"} 2023-12-19T10:35:16.150Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2829c89d-91a6-44e0-bd25-417e2c8a5cbd", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"} 2023-12-19T10:35:16.303Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2829c89d-91a6-44e0-bd25-417e2c8a5cbd", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:16.312Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2829c89d-91a6-44e0-bd25-417e2c8a5cbd", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:35:16.322Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "2829c89d-91a6-44e0-bd25-417e2c8a5cbd", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:20.942Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4a66f578-c13d-45b9-878e-704ff92acf9a", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:20.952Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4a66f578-c13d-45b9-878e-704ff92acf9a", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:20.961Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4a66f578-c13d-45b9-878e-704ff92acf9a", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:35:26.224Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a090ef45-411c-4f1b-8ff9-9c408596c9cd", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:35:26.234Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a090ef45-411c-4f1b-8ff9-9c408596c9cd", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:26.245Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "a090ef45-411c-4f1b-8ff9-9c408596c9cd", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:31.382Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "60467c1b-e4b4-4759-b7cf-68d477bf3553", "user": "monitor"} 2023-12-19T10:35:31.411Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "60467c1b-e4b4-4759-b7cf-68d477bf3553", "user": "monitor"} 2023-12-19T10:35:31.419Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "60467c1b-e4b4-4759-b7cf-68d477bf3553", "user": "monitor"} 2023-12-19T10:35:31.560Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "60467c1b-e4b4-4759-b7cf-68d477bf3553", "last-applied-secret": "81935ecf30ae523a6ee69582354f9488d47dbeb0f8c7c572af96870f2024284d"} 2023-12-19T10:35:31.676Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "60467c1b-e4b4-4759-b7cf-68d477bf3553", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:31.686Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "60467c1b-e4b4-4759-b7cf-68d477bf3553", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:35:31.702Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "60467c1b-e4b4-4759-b7cf-68d477bf3553", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:31.961Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "766a841c-e5b2-41ab-b35c-9bead9a1dc00", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:31.970Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "766a841c-e5b2-41ab-b35c-9bead9a1dc00", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:35:32.000Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "766a841c-e5b2-41ab-b35c-9bead9a1dc00", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:37.143Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b4f9ca6b-106e-4492-a5ef-e8d7eb420f84", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:35:37.154Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b4f9ca6b-106e-4492-a5ef-e8d7eb420f84", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:37.166Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b4f9ca6b-106e-4492-a5ef-e8d7eb420f84", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:42.396Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b56ab78a-5884-4eab-a0b7-b304a93adf8a", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:42.405Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b56ab78a-5884-4eab-a0b7-b304a93adf8a", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:35:42.416Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b56ab78a-5884-4eab-a0b7-b304a93adf8a", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:42.631Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "63876956-c8b9-4476-931e-329ebe0e13c0", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:42.642Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "63876956-c8b9-4476-931e-329ebe0e13c0", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:42.655Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "63876956-c8b9-4476-931e-329ebe0e13c0", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:35:47.669Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6013d03f-d423-428c-9cc2-dad3037c6dd8", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:35:47.679Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6013d03f-d423-428c-9cc2-dad3037c6dd8", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:47.703Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "6013d03f-d423-428c-9cc2-dad3037c6dd8", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:52.917Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "baecfbc7-324d-4f67-b4db-57d991406c0b", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:35:52.926Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "baecfbc7-324d-4f67-b4db-57d991406c0b", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:35:52.937Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "baecfbc7-324d-4f67-b4db-57d991406c0b", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:58.233Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b1bb10e2-2924-4560-ab7b-b0b08e8edddf", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:35:58.245Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b1bb10e2-2924-4560-ab7b-b0b08e8edddf", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:35:58.261Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "b1bb10e2-2924-4560-ab7b-b0b08e8edddf", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:36:03.451Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c3cb1b16-44d0-4623-a91d-0cfdaa3e48d9", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:36:03.461Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c3cb1b16-44d0-4623-a91d-0cfdaa3e48d9", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:36:03.472Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "c3cb1b16-44d0-4623-a91d-0cfdaa3e48d9", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:36:08.695Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "761cec34-65f3-4251-98dd-65a793d0a7a2", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:36:08.707Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "761cec34-65f3-4251-98dd-65a793d0a7a2", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:36:08.716Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "761cec34-65f3-4251-98dd-65a793d0a7a2", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:36:13.926Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "07b61927-3daa-4a9c-906a-5f28d98f052f", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:36:13.937Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "07b61927-3daa-4a9c-906a-5f28d98f052f", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:36:13.945Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "07b61927-3daa-4a9c-906a-5f28d98f052f", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:36:19.213Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "09f493ef-5ee3-4999-9d3e-61e1dfac96b8", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:36:19.222Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "09f493ef-5ee3-4999-9d3e-61e1dfac96b8", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:36:19.231Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "09f493ef-5ee3-4999-9d3e-61e1dfac96b8", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:36:24.477Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "429b12cb-c043-4e82-a0a6-6f340235c364", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:36:24.491Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "429b12cb-c043-4e82-a0a6-6f340235c364", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:36:24.503Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "429b12cb-c043-4e82-a0a6-6f340235c364", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:36:45.711Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ede08a65-dd63-44a2-837d-77bf83d9cad1", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:36:45.722Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ede08a65-dd63-44a2-837d-77bf83d9cad1", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:36:45.731Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "ede08a65-dd63-44a2-837d-77bf83d9cad1", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:36:46.372Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "29d1b382-970d-4220-b0ac-e94f84ecee7f", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:36:46.383Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "29d1b382-970d-4220-b0ac-e94f84ecee7f", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:36:46.395Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "29d1b382-970d-4220-b0ac-e94f84ecee7f", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:37:02.723Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4aa18d9a-0eda-4e1c-bd90-ccaa77ee509b", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:37:02.733Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4aa18d9a-0eda-4e1c-bd90-ccaa77ee509b", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:37:02.745Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "4aa18d9a-0eda-4e1c-bd90-ccaa77ee509b", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:37:02.923Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "151aa588-49b3-4bbe-8b55-ec6ce90ad347", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:37:02.935Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "151aa588-49b3-4bbe-8b55-ec6ce90ad347", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:37:02.945Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "151aa588-49b3-4bbe-8b55-ec6ce90ad347", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:37:07.959Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e7e5b15d-d329-44a3-90a9-e868377159b6", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:37:07.993Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e7e5b15d-d329-44a3-90a9-e868377159b6", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:37:08.003Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "e7e5b15d-d329-44a3-90a9-e868377159b6", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:37:08.212Z INFO Password updated but old one not discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f8b45853-f9a4-48f0-befc-edd1eed0a0c7", "user": "monitor"} 2023-12-19T10:37:08.569Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f8b45853-f9a4-48f0-befc-edd1eed0a0c7", "user": "monitor"} 2023-12-19T10:37:08.594Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f8b45853-f9a4-48f0-befc-edd1eed0a0c7", "last-applied-secret": "81935ecf30ae523a6ee69582354f9488d47dbeb0f8c7c572af96870f2024284d"} 2023-12-19T10:37:08.693Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f8b45853-f9a4-48f0-befc-edd1eed0a0c7", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:37:08.702Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f8b45853-f9a4-48f0-befc-edd1eed0a0c7", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} 2023-12-19T10:37:08.711Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "f8b45853-f9a4-48f0-befc-edd1eed0a0c7", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:37:13.350Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "056e515c-8d27-4225-9918-55d360c80b4a", "pod": "some-name-pxc-0", "isReplica": false, "readonly": false} 2023-12-19T10:37:13.393Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "056e515c-8d27-4225-9918-55d360c80b4a", "pod": "some-name-pxc-1", "isReplica": false, "readonly": false} 2023-12-19T10:37:13.406Z DEBUG Read only status is correct {"controller": "pxc-controller", "namespace": "users-17620", "name": "some-name", "reconcileID": "056e515c-8d27-4225-9918-55d360c80b4a", "pod": "some-name-pxc-2", "isReplica": false, "readonly": false} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.16.1/pkg/internal/controller/controller.go:227 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.16.1/pkg/internal/controller/controller.go:266 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.16.1/pkg/internal/controller/controller.go:329 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1193 [mysql] 2023/12/19 10:23:00 packets.go:37: unexpected EOF [mysql] 2023/12/19 10:34:54 packets.go:37: read tcp 10.82.249.49:37168->10.83.15.153:3306: i/o timeout sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n users-17620 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.nAGx2QF4Wu ++ mktemp + local LAST_ERR=/tmp/tmp.m327G8RQI9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nAGx2QF4Wu perconaxtradbcluster.pxc.percona.com "some-name" deleted + cat /tmp/tmp.m327G8RQI9 + rm /tmp/tmp.nAGx2QF4Wu /tmp/tmp.m327G8RQI9 + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.EjKzldYoMJ ++ mktemp + local LAST_ERR=/tmp/tmp.yfdcDFI0yi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EjKzldYoMJ No resources found + cat /tmp/tmp.yfdcDFI0yi + rm /tmp/tmp.EjKzldYoMJ /tmp/tmp.yfdcDFI0yi + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.x2UBLV3GRX ++ mktemp + local LAST_ERR=/tmp/tmp.sj2vgInSjj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.x2UBLV3GRX No resources found + cat /tmp/tmp.sj2vgInSjj + rm /tmp/tmp.x2UBLV3GRX /tmp/tmp.sj2vgInSjj + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.284noBBHeM ++ mktemp + local LAST_ERR=/tmp/tmp.97SGX9cSUX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.284noBBHeM validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.97SGX9cSUX + rm /tmp/tmp.284noBBHeM /tmp/tmp.97SGX9cSUX + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.12.1/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace users-17620 + rm -rf /tmp/tmp.TxLWXMXQ5Q + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.DohQd37ANA + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.4mWigC3FK9 test passed ----------------------------------------------------------------------------------- ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.aSj76ngd0o + local exit_status=0 + local LAST_ERR=/tmp/tmp.aNmHkNUUCF + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace users-17620 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator