Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/logs/users-8-0.log WARNING: version difference between client (1.32) and server (1.28) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.32) and server (1.28) exceeds the supported minor version skew of +/-1 + create_infra users-729 + local ns=users-729 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n users-4993 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.A80Bx28HmP ++ mktemp + local LAST_ERR=/tmp/tmp.sgcPWHxT1l + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.A80Bx28HmP perconaxtradbcluster.pxc.percona.com "some-name" deleted + cat /tmp/tmp.sgcPWHxT1l + rm /tmp/tmp.A80Bx28HmP /tmp/tmp.sgcPWHxT1l + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.82TX43kNWi ++ mktemp + local LAST_ERR=/tmp/tmp.vYjZGQA2oh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.82TX43kNWi No resources found + cat /tmp/tmp.vYjZGQA2oh + rm /tmp/tmp.82TX43kNWi /tmp/tmp.vYjZGQA2oh + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.PHLuma9o35 ++ mktemp + local LAST_ERR=/tmp/tmp.LnXQkScySI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PHLuma9o35 No resources found + cat /tmp/tmp.LnXQkScySI + rm /tmp/tmp.PHLuma9o35 /tmp/tmp.LnXQkScySI + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' ++ mktemp + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + local LAST_OUT=/tmp/tmp.AHvKXs81nA + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.RqY5hpMUeK ++ mktemp + local LAST_ERR=/tmp/tmp.1HU7y8zQCo + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.z3iB5ytBlu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AHvKXs81nA + cat /tmp/tmp.1HU7y8zQCo + rm /tmp/tmp.AHvKXs81nA /tmp/tmp.1HU7y8zQCo + return 0 namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted namespace "users-4993" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RqY5hpMUeK namespace "pxc-operator" deleted + cat /tmp/tmp.z3iB5ytBlu + rm /tmp/tmp.RqY5hpMUeK /tmp/tmp.z3iB5ytBlu + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.N0ldHIQVLt ++ mktemp + local LAST_ERR=/tmp/tmp.QiDHWbh7l1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.N0ldHIQVLt namespace/pxc-operator created + cat /tmp/tmp.QiDHWbh7l1 + rm /tmp/tmp.N0ldHIQVLt /tmp/tmp.QiDHWbh7l1 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.SED9sn9Spc +++ mktemp ++ local LAST_ERR=/tmp/tmp.r4Q6J1f3e9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SED9sn9Spc ++ cat /tmp/tmp.r4Q6J1f3e9 ++ rm /tmp/tmp.SED9sn9Spc /tmp/tmp.r4Q6J1f3e9 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1948-70c11cfc-1-cluster6 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ufnsUWB8lF ++ mktemp + local LAST_ERR=/tmp/tmp.qcB7i0fqFr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1948-70c11cfc-1-cluster6 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ufnsUWB8lF Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1948-70c11cfc-1-cluster6" modified. + cat /tmp/tmp.qcB7i0fqFr + rm /tmp/tmp.ufnsUWB8lF /tmp/tmp.qcB7i0fqFr + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.PTeqzUw0yu ++ mktemp + local LAST_ERR=/tmp/tmp.mZMBAF9EUB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PTeqzUw0yu customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.mZMBAF9EUB + rm /tmp/tmp.PTeqzUw0yu /tmp/tmp.mZMBAF9EUB + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.xHCNpx4LbT ++ mktemp + local LAST_ERR=/tmp/tmp.4tbxJ5kYtP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xHCNpx4LbT clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.4tbxJ5kYtP + rm /tmp/tmp.xHCNpx4LbT /tmp/tmp.4tbxJ5kYtP + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1948-70c11cfc^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - ++ mktemp + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - + local LAST_OUT=/tmp/tmp.yFfgtRYuVN ++ mktemp + local LAST_ERR=/tmp/tmp.gAFGmwdKR0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yFfgtRYuVN deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.gAFGmwdKR0 + rm /tmp/tmp.yFfgtRYuVN /tmp/tmp.gAFGmwdKR0 + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.oOMnSwFsuY ++ mktemp + local LAST_ERR=/tmp/tmp.SRCFE8G7i5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oOMnSwFsuY pod/percona-xtradb-cluster-operator-5f5f9cbb4-fvlrw condition met + cat /tmp/tmp.SRCFE8G7i5 + rm /tmp/tmp.oOMnSwFsuY /tmp/tmp.SRCFE8G7i5 + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.PXQZBoSMmd +++ mktemp ++ local LAST_ERR=/tmp/tmp.mZkYGPYJBs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PXQZBoSMmd ++ cat /tmp/tmp.mZkYGPYJBs ++ rm /tmp/tmp.PXQZBoSMmd /tmp/tmp.mZkYGPYJBs ++ return 0 + wait_pod percona-xtradb-cluster-operator-5f5f9cbb4-fvlrw 480 pxc-operator + local pod=percona-xtradb-cluster-operator-5f5f9cbb4-fvlrw + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-5f5f9cbb4-fvlrw ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-5f5f9cbb4-fvlrw condition met waiting for pod/percona-xtradb-cluster-operator-5f5f9cbb4-fvlrw to become Ready.Ok + sleep 3 + create_namespace users-729 + local namespace=users-729 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + '[' -n '' ']' + xargs kubectl delete ns + desc 'cleaned up old namespaces users-729' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces users-729 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace users-729 ++ mktemp ++ mktemp + awk '{print$1}' + local LAST_OUT=/tmp/tmp.THrJevRUuo + local LAST_OUT=/tmp/tmp.aDKtXti97P ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.KQYmAWyTti + local exit_status=0 + local LAST_ERR=/tmp/tmp.cdpbwewFHJ + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace users-729 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace users-729 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.THrJevRUuo + cat /tmp/tmp.cdpbwewFHJ + rm /tmp/tmp.THrJevRUuo /tmp/tmp.cdpbwewFHJ + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace users-729 namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.aDKtXti97P + cat /tmp/tmp.KQYmAWyTti Error from server (NotFound): namespaces "users-729" not found + rm /tmp/tmp.aDKtXti97P /tmp/tmp.KQYmAWyTti + return 1 + : + wait_for_delete namespace/users-729 + local res=namespace/users-729 + echo -n 'waiting for namespace/users-729 to be deleted' waiting for namespace/users-729 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "users-729" not found + desc 'create namespace users-729' + set +o xtrace ----------------------------------------------------------------------------------- create namespace users-729 ----------------------------------------------------------------------------------- + kubectl_bin create namespace users-729 ++ mktemp + local LAST_OUT=/tmp/tmp.5JrALGBX7A ++ mktemp + local LAST_ERR=/tmp/tmp.8DUhg6jMQE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace users-729 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5JrALGBX7A namespace/users-729 created + cat /tmp/tmp.8DUhg6jMQE + rm /tmp/tmp.5JrALGBX7A /tmp/tmp.8DUhg6jMQE + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.g4ai1JYyqv +++ mktemp ++ local LAST_ERR=/tmp/tmp.X0UCx4MvOz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.g4ai1JYyqv ++ cat /tmp/tmp.X0UCx4MvOz ++ rm /tmp/tmp.g4ai1JYyqv /tmp/tmp.X0UCx4MvOz ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1948-70c11cfc-1-cluster6 --namespace=users-729 ++ mktemp + local LAST_OUT=/tmp/tmp.isGQOmb8U7 ++ mktemp + local LAST_ERR=/tmp/tmp.CdOvh2DDXf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1948-70c11cfc-1-cluster6 --namespace=users-729 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.isGQOmb8U7 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1948-70c11cfc-1-cluster6" modified. + cat /tmp/tmp.CdOvh2DDXf + rm /tmp/tmp.isGQOmb8U7 /tmp/tmp.CdOvh2DDXf + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.hSlOcDNfrl ++ mktemp + local LAST_ERR=/tmp/tmp.ouaDIuuDGt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hSlOcDNfrl secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.ouaDIuuDGt + rm /tmp/tmp.hSlOcDNfrl /tmp/tmp.ouaDIuuDGt + return 0 + desc 'create PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC cluster ----------------------------------------------------------------------------------- + newpass=test-password ++ echo -n test-password ++ base64 + newpassencrypted=dGVzdC1wYXNzd29yZA== + cluster=some-name + spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/some-name.yml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/some-name.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.EmWf4oFtNG ++ mktemp + local LAST_ERR=/tmp/tmp.Kh1ZwKKJFd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EmWf4oFtNG secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.Kh1ZwKKJFd + rm /tmp/tmp.EmWf4oFtNG /tmp/tmp.Kh1ZwKKJFd + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-729~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_OUT=/tmp/tmp.pjjhXFzKMj + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1948-70c11cfc#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_ERR=/tmp/tmp.DDc68aPkl7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pjjhXFzKMj deployment.apps/pxc-client created + cat /tmp/tmp.DDc68aPkl7 + rm /tmp/tmp.pjjhXFzKMj /tmp/tmp.DDc68aPkl7 + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/conf/some-name.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1948-70c11cfc#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.vnun5uo5PA + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-729~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.hAJPm3KTzh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vnun5uo5PA perconaxtradbcluster.pxc.percona.com/some-name created + cat /tmp/tmp.hAJPm3KTzh + rm /tmp/tmp.vnun5uo5PA /tmp/tmp.hAJPm3KTzh + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LcQgFtKaN6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Qj9W4lNxx4 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.LcQgFtKaN6 +++ cat /tmp/tmp.Qj9W4lNxx4 +++ rm /tmp/tmp.LcQgFtKaN6 /tmp/tmp.Qj9W4lNxx4 +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zwRZDFpD9A ++++ mktemp +++ local LAST_ERR=/tmp/tmp.O8VQvGhFrd +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zwRZDFpD9A +++ cat /tmp/tmp.O8VQvGhFrd +++ rm /tmp/tmp.zwRZDFpD9A /tmp/tmp.O8VQvGhFrd +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo some-name-proxysql ++ return + local proxy=some-name-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-729 ++ mktemp + local LAST_OUT=/tmp/tmp.bOG4aIobYI ++ mktemp + local LAST_ERR=/tmp/tmp.HY9lfDnvyU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-729 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-729 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-729 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.bOG4aIobYI + cat /tmp/tmp.HY9lfDnvyU error: no matching resources found + rm /tmp/tmp.bOG4aIobYI /tmp/tmp.HY9lfDnvyU + return 1 + true + wait_for_running some-name-proxysql 1 + local name=some-name-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-proxysql-0 480 + local pod=some-name-proxysql-0 + local max_retry=480 + local ns= ++ echo some-name-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/some-name-proxysql-0 condition met waiting for pod/some-name-proxysql-0 to become Ready.Ok + wait_for_running some-name-pxc 3 + local name=some-name-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ echo some-name-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met waiting for pod/some-name-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-1 480 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-2 480 + local pod=some-name-pxc-2 + local max_retry=480 + local ns= ++ echo some-name-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-2 condition met waiting for pod/some-name-pxc-2 to become Ready.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-proxysql -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h some-name-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eZWKWxj8fD +++ mktemp ++ local LAST_ERR=/tmp/tmp.rOPZtbR1a1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eZWKWxj8fD ++ cat /tmp/tmp.rOPZtbR1a1 ++ rm /tmp/tmp.eZWKWxj8fD /tmp/tmp.rOPZtbR1a1 ++ return 0 + client_pod=pxc-client-69f9bd646c-95zbk + wait_pod pxc-client-69f9bd646c-95zbk + local pod=pxc-client-69f9bd646c-95zbk + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-95zbk ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-95zbk condition met waiting for pod/pxc-client-69f9bd646c-95zbk to become Ready.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-proxysql -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h some-name-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qx3cvb2U5U +++ mktemp ++ local LAST_ERR=/tmp/tmp.QR3MxyY7Qt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qx3cvb2U5U ++ cat /tmp/tmp.QR3MxyY7Qt ++ rm /tmp/tmp.qx3cvb2U5U /tmp/tmp.QR3MxyY7Qt ++ return 0 + client_pod=pxc-client-69f9bd646c-95zbk + wait_pod pxc-client-69f9bd646c-95zbk + local pod=pxc-client-69f9bd646c-95zbk + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-95zbk ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-95zbk condition met waiting for pod/pxc-client-69f9bd646c-95zbk to become Ready.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ah24WOG6Gv +++ mktemp ++ local LAST_ERR=/tmp/tmp.BML0oBM3jb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ah24WOG6Gv ++ cat /tmp/tmp.BML0oBM3jb ++ rm /tmp/tmp.ah24WOG6Gv /tmp/tmp.BML0oBM3jb ++ return 0 + client_pod=pxc-client-69f9bd646c-95zbk + wait_pod pxc-client-69f9bd646c-95zbk + local pod=pxc-client-69f9bd646c-95zbk + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-95zbk ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-95zbk condition met waiting for pod/pxc-client-69f9bd646c-95zbk to become Ready.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-1.sql /tmp/tmp.HcH8ZcrDqq/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xdHRjKoASn +++ mktemp ++ local LAST_ERR=/tmp/tmp.JvHUWqwT29 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xdHRjKoASn ++ cat /tmp/tmp.JvHUWqwT29 ++ rm /tmp/tmp.xdHRjKoASn /tmp/tmp.JvHUWqwT29 ++ return 0 + client_pod=pxc-client-69f9bd646c-95zbk + wait_pod pxc-client-69f9bd646c-95zbk + local pod=pxc-client-69f9bd646c-95zbk + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-95zbk ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-95zbk condition met waiting for pod/pxc-client-69f9bd646c-95zbk to become Ready.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-1.sql /tmp/tmp.HcH8ZcrDqq/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lArmD3Sn6l +++ mktemp ++ local LAST_ERR=/tmp/tmp.9Y5MYMmIXQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lArmD3Sn6l ++ cat /tmp/tmp.9Y5MYMmIXQ ++ rm /tmp/tmp.lArmD3Sn6l /tmp/tmp.9Y5MYMmIXQ ++ return 0 + client_pod=pxc-client-69f9bd646c-95zbk + wait_pod pxc-client-69f9bd646c-95zbk + local pod=pxc-client-69f9bd646c-95zbk + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-95zbk ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-95zbk condition met waiting for pod/pxc-client-69f9bd646c-95zbk to become Ready.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-1.sql /tmp/tmp.HcH8ZcrDqq/select-1.sql ++ is_keyring_plugin_in_use some-name ++ local cluster=some-name ++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kPAUckkeYH +++ mktemp ++ local LAST_ERR=/tmp/tmp.NztlZ2Kpbq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kPAUckkeYH ++ cat /tmp/tmp.NztlZ2Kpbq Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.kPAUckkeYH /tmp/tmp.NztlZ2Kpbq ++ return 0 + '[' '' ']' + desc 'test root' + set +o xtrace ----------------------------------------------------------------------------------- test root ----------------------------------------------------------------------------------- + patch_secret my-cluster-secrets root dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=root + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.h9YK3dM8y9 ++ mktemp + local LAST_ERR=/tmp/tmp.LplxuVWS9I + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.h9YK3dM8y9 secret/my-cluster-secrets patched + cat /tmp/tmp.LplxuVWS9I + rm /tmp/tmp.h9YK3dM8y9 /tmp/tmp.LplxuVWS9I + return 0 + sleep 15 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql ]] + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.URg9xEFnBk +++ mktemp ++ local LAST_ERR=/tmp/tmp.RANI4CA7rg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.URg9xEFnBk ++ cat /tmp/tmp.RANI4CA7rg ++ rm /tmp/tmp.URg9xEFnBk /tmp/tmp.RANI4CA7rg ++ return 0 + client_pod=pxc-client-69f9bd646c-95zbk + wait_pod pxc-client-69f9bd646c-95zbk + local pod=pxc-client-69f9bd646c-95zbk + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-95zbk ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-95zbk condition met waiting for pod/pxc-client-69f9bd646c-95zbk to become Ready.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.HcH8ZcrDqq/select-4.sql + desc 'test proxyadmin' + set +o xtrace ----------------------------------------------------------------------------------- test proxyadmin ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.DQFXAvMPSF ++ mktemp + local LAST_ERR=/tmp/tmp.vxsbMCMvJw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DQFXAvMPSF perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.vxsbMCMvJw + rm /tmp/tmp.DQFXAvMPSF /tmp/tmp.vxsbMCMvJw + return 0 + sleep 15 + wait_cluster_consistency some-name 3 3 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=3 + '[' -z 3 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FLHq8xgrtu +++ mktemp ++ local LAST_ERR=/tmp/tmp.pqjAASsGm4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FLHq8xgrtu ++ cat /tmp/tmp.pqjAASsGm4 ++ rm /tmp/tmp.FLHq8xgrtu /tmp/tmp.pqjAASsGm4 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bY62jeuSVh +++ mktemp ++ local LAST_ERR=/tmp/tmp.R7ZUJtOBbk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bY62jeuSVh ++ cat /tmp/tmp.R7ZUJtOBbk ++ rm /tmp/tmp.bY62jeuSVh /tmp/tmp.R7ZUJtOBbk ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Yr3zET1zVh +++ mktemp ++ local LAST_ERR=/tmp/tmp.dZeFojDqNc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Yr3zET1zVh ++ cat /tmp/tmp.dZeFojDqNc ++ rm /tmp/tmp.Yr3zET1zVh /tmp/tmp.dZeFojDqNc ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.lTPghnalR5 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.fDj1XS5XMm +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.lTPghnalR5 +++++ cat /tmp/tmp.fDj1XS5XMm +++++ rm /tmp/tmp.lTPghnalR5 /tmp/tmp.fDj1XS5XMm +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.nstr2O4s93 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.TR8GF6D0MN +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.nstr2O4s93 +++++ cat /tmp/tmp.TR8GF6D0MN +++++ rm /tmp/tmp.nstr2O4s93 /tmp/tmp.TR8GF6D0MN +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6N4SNhRCBK +++ mktemp ++ local LAST_ERR=/tmp/tmp.NwOrrT5f5w ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6N4SNhRCBK ++ cat /tmp/tmp.NwOrrT5f5w ++ rm /tmp/tmp.6N4SNhRCBK /tmp/tmp.NwOrrT5f5w ++ return 0 + [[ 3 == \3 ]] + patch_secret my-cluster-secrets proxyadmin dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=proxyadmin + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.OeAYsKKKFu ++ mktemp + local LAST_ERR=/tmp/tmp.xSuDGkz3VR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OeAYsKKKFu secret/my-cluster-secrets patched + cat /tmp/tmp.xSuDGkz3VR + rm /tmp/tmp.OeAYsKKKFu /tmp/tmp.xSuDGkz3VR + return 0 + sleep 15 + wait_cluster_consistency some-name 3 3 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=3 + '[' -z 3 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Cm1dPRCWtW +++ mktemp ++ local LAST_ERR=/tmp/tmp.2GtpnmmmTE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Cm1dPRCWtW ++ cat /tmp/tmp.2GtpnmmmTE ++ rm /tmp/tmp.Cm1dPRCWtW /tmp/tmp.2GtpnmmmTE ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1SbD36c1Ji +++ mktemp ++ local LAST_ERR=/tmp/tmp.mKsfDVLEoy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1SbD36c1Ji ++ cat /tmp/tmp.mKsfDVLEoy ++ rm /tmp/tmp.1SbD36c1Ji /tmp/tmp.mKsfDVLEoy ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pfdJO7uCzk +++ mktemp ++ local LAST_ERR=/tmp/tmp.fKFfMJKHY6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pfdJO7uCzk ++ cat /tmp/tmp.fKFfMJKHY6 ++ rm /tmp/tmp.pfdJO7uCzk /tmp/tmp.fKFfMJKHY6 ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.p35YWQcsEc ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.SUc3fDePJu +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.p35YWQcsEc +++++ cat /tmp/tmp.SUc3fDePJu +++++ rm /tmp/tmp.p35YWQcsEc /tmp/tmp.SUc3fDePJu +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.d2l4yQGZMz ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ErypNtSl0Z +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.d2l4yQGZMz +++++ cat /tmp/tmp.ErypNtSl0Z +++++ rm /tmp/tmp.d2l4yQGZMz /tmp/tmp.ErypNtSl0Z +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xVVUNqF3Qr +++ mktemp ++ local LAST_ERR=/tmp/tmp.Gcr5O5pimT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xVVUNqF3Qr ++ cat /tmp/tmp.Gcr5O5pimT ++ rm /tmp/tmp.xVVUNqF3Qr /tmp/tmp.Gcr5O5pimT ++ return 0 + [[ 3 == \3 ]] + compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 '' proxysql + local command_id=select-2 + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-0 + local postfix= + local container_name=proxysql + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-2-80.sql ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 proxysql + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-0 + local container_name=proxysql + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-2.sql /tmp/tmp.HcH8ZcrDqq/select-2.sql + compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 '' proxysql + local command_id=select-2 + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-1 + local postfix= + local container_name=proxysql + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-2-80.sql ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 proxysql + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-1 + local container_name=proxysql + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-2.sql /tmp/tmp.HcH8ZcrDqq/select-2.sql + compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 '' proxysql + local command_id=select-2 + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-2 + local postfix= + local container_name=proxysql + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-2-80.sql ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 proxysql + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-2 + local container_name=proxysql + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-2.sql /tmp/tmp.HcH8ZcrDqq/select-2.sql + desc 'test xtrabackup' + set +o xtrace ----------------------------------------------------------------------------------- test xtrabackup ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.pz0X0P2PYZ ++ mktemp + local LAST_ERR=/tmp/tmp.aqW1VA9p71 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pz0X0P2PYZ perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.aqW1VA9p71 + rm /tmp/tmp.pz0X0P2PYZ /tmp/tmp.aqW1VA9p71 + return 0 + patch_secret my-cluster-secrets xtrabackup dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=xtrabackup + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.cXB6Su9Apg ++ mktemp + local LAST_ERR=/tmp/tmp.wKcdipUyJc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cXB6Su9Apg secret/my-cluster-secrets patched + cat /tmp/tmp.wKcdipUyJc + rm /tmp/tmp.cXB6Su9Apg /tmp/tmp.wKcdipUyJc + return 0 + sleep 15 + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yK5zEc8ELq +++ mktemp ++ local LAST_ERR=/tmp/tmp.HReQAhcUT7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yK5zEc8ELq ++ cat /tmp/tmp.HReQAhcUT7 ++ rm /tmp/tmp.yK5zEc8ELq /tmp/tmp.HReQAhcUT7 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rhtxCS4idI +++ mktemp ++ local LAST_ERR=/tmp/tmp.IDXf4kLnSj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rhtxCS4idI ++ cat /tmp/tmp.IDXf4kLnSj ++ rm /tmp/tmp.rhtxCS4idI /tmp/tmp.IDXf4kLnSj ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UrwtLqLpdW +++ mktemp ++ local LAST_ERR=/tmp/tmp.FcZAMaWasb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UrwtLqLpdW ++ cat /tmp/tmp.FcZAMaWasb ++ rm /tmp/tmp.UrwtLqLpdW /tmp/tmp.FcZAMaWasb ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KAKlJulkEL +++ mktemp ++ local LAST_ERR=/tmp/tmp.M3M0DQl6DE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KAKlJulkEL ++ cat /tmp/tmp.M3M0DQl6DE ++ rm /tmp/tmp.KAKlJulkEL /tmp/tmp.M3M0DQl6DE ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wtHOD9JFtY +++ mktemp ++ local LAST_ERR=/tmp/tmp.iEJMx5c8gV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wtHOD9JFtY ++ cat /tmp/tmp.iEJMx5c8gV ++ rm /tmp/tmp.wtHOD9JFtY /tmp/tmp.iEJMx5c8gV ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.L00394pMi0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.roBesVeOwW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.L00394pMi0 ++ cat /tmp/tmp.roBesVeOwW ++ rm /tmp/tmp.L00394pMi0 /tmp/tmp.roBesVeOwW ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MHYCIi9LJZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.wd0KJ855Rz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MHYCIi9LJZ ++ cat /tmp/tmp.wd0KJ855Rz ++ rm /tmp/tmp.MHYCIi9LJZ /tmp/tmp.wd0KJ855Rz ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JXdSy9Th0w +++ mktemp ++ local LAST_ERR=/tmp/tmp.1ifF8oTXed ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JXdSy9Th0w ++ cat /tmp/tmp.1ifF8oTXed ++ rm /tmp/tmp.JXdSy9Th0w /tmp/tmp.1ifF8oTXed ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.chrWmmPMjG +++ mktemp ++ local LAST_ERR=/tmp/tmp.4eKXzIRcos ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.chrWmmPMjG ++ cat /tmp/tmp.4eKXzIRcos ++ rm /tmp/tmp.chrWmmPMjG /tmp/tmp.4eKXzIRcos ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 8 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.j5cQVTqvzh +++ mktemp ++ local LAST_ERR=/tmp/tmp.C1Dh4C3qFD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.j5cQVTqvzh ++ cat /tmp/tmp.C1Dh4C3qFD ++ rm /tmp/tmp.j5cQVTqvzh /tmp/tmp.C1Dh4C3qFD ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 9 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2auy9vwADn +++ mktemp ++ local LAST_ERR=/tmp/tmp.84pun6BIZ8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2auy9vwADn ++ cat /tmp/tmp.84pun6BIZ8 ++ rm /tmp/tmp.2auy9vwADn /tmp/tmp.84pun6BIZ8 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 10 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mpsRYDi6Q1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.K9GX0HNF0n ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mpsRYDi6Q1 ++ cat /tmp/tmp.K9GX0HNF0n ++ rm /tmp/tmp.mpsRYDi6Q1 /tmp/tmp.K9GX0HNF0n ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 11 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZZVX4fFvkR +++ mktemp ++ local LAST_ERR=/tmp/tmp.cB8OH4Tx9F ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZZVX4fFvkR ++ cat /tmp/tmp.cB8OH4Tx9F ++ rm /tmp/tmp.ZZVX4fFvkR /tmp/tmp.cB8OH4Tx9F ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 12 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mXyTG1GUSw +++ mktemp ++ local LAST_ERR=/tmp/tmp.2KkoO3MKFj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mXyTG1GUSw ++ cat /tmp/tmp.2KkoO3MKFj ++ rm /tmp/tmp.mXyTG1GUSw /tmp/tmp.2KkoO3MKFj ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 13 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.p2WukKIW5D +++ mktemp ++ local LAST_ERR=/tmp/tmp.dMYYeC9iqa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.p2WukKIW5D ++ cat /tmp/tmp.dMYYeC9iqa ++ rm /tmp/tmp.p2WukKIW5D /tmp/tmp.dMYYeC9iqa ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 14 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.T75gYTlCK3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.INNkXgv3ZC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.T75gYTlCK3 ++ cat /tmp/tmp.INNkXgv3ZC ++ rm /tmp/tmp.T75gYTlCK3 /tmp/tmp.INNkXgv3ZC ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 15 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cUXEykrzbZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.jlJjO55UM5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cUXEykrzbZ ++ cat /tmp/tmp.jlJjO55UM5 ++ rm /tmp/tmp.cUXEykrzbZ /tmp/tmp.jlJjO55UM5 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 16 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pa3oHCrWic +++ mktemp ++ local LAST_ERR=/tmp/tmp.61Nj70tfEI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pa3oHCrWic ++ cat /tmp/tmp.61Nj70tfEI ++ rm /tmp/tmp.pa3oHCrWic /tmp/tmp.61Nj70tfEI ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 17 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.apd3BFqvA9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RPVtC4zdpA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.apd3BFqvA9 ++ cat /tmp/tmp.RPVtC4zdpA ++ rm /tmp/tmp.apd3BFqvA9 /tmp/tmp.RPVtC4zdpA ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 18 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GCbUvMaAG7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.78grz646us ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GCbUvMaAG7 ++ cat /tmp/tmp.78grz646us ++ rm /tmp/tmp.GCbUvMaAG7 /tmp/tmp.78grz646us ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 19 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UFElT4f40t +++ mktemp ++ local LAST_ERR=/tmp/tmp.BRqERficGU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UFElT4f40t ++ cat /tmp/tmp.BRqERficGU ++ rm /tmp/tmp.UFElT4f40t /tmp/tmp.BRqERficGU ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 20 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aV1MMl977L +++ mktemp ++ local LAST_ERR=/tmp/tmp.W7xXpOaff0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aV1MMl977L ++ cat /tmp/tmp.W7xXpOaff0 ++ rm /tmp/tmp.aV1MMl977L /tmp/tmp.W7xXpOaff0 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 21 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qbs7STcHSU +++ mktemp ++ local LAST_ERR=/tmp/tmp.SbS9QYQrB1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qbs7STcHSU ++ cat /tmp/tmp.SbS9QYQrB1 ++ rm /tmp/tmp.qbs7STcHSU /tmp/tmp.SbS9QYQrB1 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 22 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.e3uTeMVRej +++ mktemp ++ local LAST_ERR=/tmp/tmp.xgKoMNPIsR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.e3uTeMVRej ++ cat /tmp/tmp.xgKoMNPIsR ++ rm /tmp/tmp.e3uTeMVRej /tmp/tmp.xgKoMNPIsR ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 23 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RZ5P9nDDIv +++ mktemp ++ local LAST_ERR=/tmp/tmp.8haOc5TRXK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RZ5P9nDDIv ++ cat /tmp/tmp.8haOc5TRXK ++ rm /tmp/tmp.RZ5P9nDDIv /tmp/tmp.8haOc5TRXK ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 24 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2wKDwboidP +++ mktemp ++ local LAST_ERR=/tmp/tmp.2tdKC22P0w ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2wKDwboidP ++ cat /tmp/tmp.2tdKC22P0w ++ rm /tmp/tmp.2wKDwboidP /tmp/tmp.2tdKC22P0w ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 25 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DLK75LfzrU +++ mktemp ++ local LAST_ERR=/tmp/tmp.mvuzzT5kTv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DLK75LfzrU ++ cat /tmp/tmp.mvuzzT5kTv ++ rm /tmp/tmp.DLK75LfzrU /tmp/tmp.mvuzzT5kTv ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 26 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XQ47jpI1b9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.yZvCYOaory ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XQ47jpI1b9 ++ cat /tmp/tmp.yZvCYOaory ++ rm /tmp/tmp.XQ47jpI1b9 /tmp/tmp.yZvCYOaory ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 27 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XtTgzsY0A5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JchC7It0nj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XtTgzsY0A5 ++ cat /tmp/tmp.JchC7It0nj ++ rm /tmp/tmp.XtTgzsY0A5 /tmp/tmp.JchC7It0nj ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 28 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lFTzqKdTo8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.fvEnRhmmVS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lFTzqKdTo8 ++ cat /tmp/tmp.fvEnRhmmVS ++ rm /tmp/tmp.lFTzqKdTo8 /tmp/tmp.fvEnRhmmVS ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 29 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.G7bIiDH11z +++ mktemp ++ local LAST_ERR=/tmp/tmp.Sc9fryFTsR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.G7bIiDH11z ++ cat /tmp/tmp.Sc9fryFTsR ++ rm /tmp/tmp.G7bIiDH11z /tmp/tmp.Sc9fryFTsR ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 30 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SZFM6ndiRv +++ mktemp ++ local LAST_ERR=/tmp/tmp.55MMnoAofU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SZFM6ndiRv ++ cat /tmp/tmp.55MMnoAofU ++ rm /tmp/tmp.SZFM6ndiRv /tmp/tmp.55MMnoAofU ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 31 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.exSsuFcofV +++ mktemp ++ local LAST_ERR=/tmp/tmp.XCSiuiOdfe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.exSsuFcofV ++ cat /tmp/tmp.XCSiuiOdfe ++ rm /tmp/tmp.exSsuFcofV /tmp/tmp.XCSiuiOdfe ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 32 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BiQGR0LfHr +++ mktemp ++ local LAST_ERR=/tmp/tmp.lk7ziCbAZM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BiQGR0LfHr ++ cat /tmp/tmp.lk7ziCbAZM ++ rm /tmp/tmp.BiQGR0LfHr /tmp/tmp.lk7ziCbAZM ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 33 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n8GzZnTdXo +++ mktemp ++ local LAST_ERR=/tmp/tmp.AKJS8why5A ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.n8GzZnTdXo ++ cat /tmp/tmp.AKJS8why5A ++ rm /tmp/tmp.n8GzZnTdXo /tmp/tmp.AKJS8why5A ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gv5MoNvBhu +++ mktemp ++ local LAST_ERR=/tmp/tmp.V0EL9ymJ0g ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gv5MoNvBhu ++ cat /tmp/tmp.V0EL9ymJ0g ++ rm /tmp/tmp.gv5MoNvBhu /tmp/tmp.V0EL9ymJ0g ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.3UzuKOOjmF ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.WsQ9XBQSLV +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.3UzuKOOjmF +++++ cat /tmp/tmp.WsQ9XBQSLV +++++ rm /tmp/tmp.3UzuKOOjmF /tmp/tmp.WsQ9XBQSLV +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ifRlaArV8t ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.20Mx4Kyd5y +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ifRlaArV8t +++++ cat /tmp/tmp.20Mx4Kyd5y +++++ rm /tmp/tmp.ifRlaArV8t /tmp/tmp.20Mx4Kyd5y +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QVPaxufHqG +++ mktemp ++ local LAST_ERR=/tmp/tmp.JwnYgR1kM5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QVPaxufHqG ++ cat /tmp/tmp.JwnYgR1kM5 ++ rm /tmp/tmp.QVPaxufHqG /tmp/tmp.JwnYgR1kM5 ++ return 0 + [[ 2 == \2 ]] + compare_mysql_cmd_local select-3 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 '' pxc + local command_id=select-3 + local 'command=SHOW DATABASES;' + local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' + local pod=some-name-pxc-0 + local postfix= + local container_name=pxc + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-3-80.sql ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + run_mysql_local 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 pxc + local 'command=SHOW DATABASES;' + local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' + local pod=some-name-pxc-0 + local container_name=pxc + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-3.sql /tmp/tmp.HcH8ZcrDqq/select-3.sql + desc 'test monitor' + set +o xtrace ----------------------------------------------------------------------------------- test monitor ----------------------------------------------------------------------------------- + patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=monitor + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.KvwQWtPZtf ++ mktemp + local LAST_ERR=/tmp/tmp.WDjNoZ6hjH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KvwQWtPZtf secret/my-cluster-secrets patched + cat /tmp/tmp.WDjNoZ6hjH + rm /tmp/tmp.KvwQWtPZtf /tmp/tmp.WDjNoZ6hjH + return 0 + wait_for_password_propagation my-cluster-secrets monitor + local secret=my-cluster-secrets + local user=monitor + local max_retry=240 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.zNiGBGqotA +++ mktemp ++ local LAST_ERR=/tmp/tmp.Zy7e0Dloqb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zNiGBGqotA ++ cat /tmp/tmp.Zy7e0Dloqb ++ rm /tmp/tmp.zNiGBGqotA /tmp/tmp.Zy7e0Dloqb ++ return 0 + local root_pass=test-password + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + retry=0 + is_password_updated monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + grep additional_password + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OrR11kbmqy +++ mktemp ++ local LAST_ERR=/tmp/tmp.Q6DxNfMLcW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OrR11kbmqy ++ cat /tmp/tmp.Q6DxNfMLcW ++ rm /tmp/tmp.OrR11kbmqy /tmp/tmp.Q6DxNfMLcW ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace {"additional_password": "*C1F414D9BAF378B656A849B31F9F8AF3125F558B"} + retry=0 + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9j1WWvOZ3D +++ mktemp ++ local LAST_ERR=/tmp/tmp.mSJGl0cr6n ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9j1WWvOZ3D ++ cat /tmp/tmp.mSJGl0cr6n ++ rm /tmp/tmp.9j1WWvOZ3D /tmp/tmp.mSJGl0cr6n ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 1 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + grep NULL + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UmqcpYjc0b +++ mktemp ++ local LAST_ERR=/tmp/tmp.LAJvktO59m ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UmqcpYjc0b ++ cat /tmp/tmp.LAJvktO59m ++ rm /tmp/tmp.UmqcpYjc0b /tmp/tmp.LAJvktO59m ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 2 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nG7hgNcw3D +++ mktemp ++ local LAST_ERR=/tmp/tmp.KRuEX0whyx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nG7hgNcw3D ++ cat /tmp/tmp.KRuEX0whyx ++ rm /tmp/tmp.nG7hgNcw3D /tmp/tmp.KRuEX0whyx ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 3 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MXDhsEbDZd +++ mktemp ++ local LAST_ERR=/tmp/tmp.av8CxjiKSs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MXDhsEbDZd ++ cat /tmp/tmp.av8CxjiKSs ++ rm /tmp/tmp.MXDhsEbDZd /tmp/tmp.av8CxjiKSs ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 4 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.od0oWN4Uoo +++ mktemp ++ local LAST_ERR=/tmp/tmp.zy7EpYq7WM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.od0oWN4Uoo ++ cat /tmp/tmp.zy7EpYq7WM ++ rm /tmp/tmp.od0oWN4Uoo /tmp/tmp.zy7EpYq7WM ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 5 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0g5rFNm1sZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.zeCc0F8vnv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0g5rFNm1sZ ++ cat /tmp/tmp.zeCc0F8vnv ++ rm /tmp/tmp.0g5rFNm1sZ /tmp/tmp.zeCc0F8vnv ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 6 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kdM7ky3mw0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.aUxU7nS7EX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kdM7ky3mw0 ++ cat /tmp/tmp.aUxU7nS7EX ++ rm /tmp/tmp.kdM7ky3mw0 /tmp/tmp.aUxU7nS7EX ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 7 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + grep NULL + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.x0g4Gmt6tU +++ mktemp ++ local LAST_ERR=/tmp/tmp.tJ2NQpvZjb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.x0g4Gmt6tU ++ cat /tmp/tmp.tJ2NQpvZjb ++ rm /tmp/tmp.x0g4Gmt6tU /tmp/tmp.tJ2NQpvZjb ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 8 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0UwcaQu3uY +++ mktemp ++ local LAST_ERR=/tmp/tmp.Qlq5Cmgbey ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0UwcaQu3uY ++ cat /tmp/tmp.Qlq5Cmgbey ++ rm /tmp/tmp.0UwcaQu3uY /tmp/tmp.Qlq5Cmgbey ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 9 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.W0MiS5pmpy +++ mktemp ++ local LAST_ERR=/tmp/tmp.Pk9iJ59j3h ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.W0MiS5pmpy ++ cat /tmp/tmp.Pk9iJ59j3h ++ rm /tmp/tmp.W0MiS5pmpy /tmp/tmp.Pk9iJ59j3h ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 10 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + grep NULL + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qLWCaPMjhC +++ mktemp ++ local LAST_ERR=/tmp/tmp.QFSztExGS7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qLWCaPMjhC ++ cat /tmp/tmp.QFSztExGS7 ++ rm /tmp/tmp.qLWCaPMjhC /tmp/tmp.QFSztExGS7 ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 11 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + grep NULL + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vBiUd0VkWK +++ mktemp ++ local LAST_ERR=/tmp/tmp.ByTAbvMtBN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vBiUd0VkWK ++ cat /tmp/tmp.ByTAbvMtBN ++ rm /tmp/tmp.vBiUd0VkWK /tmp/tmp.ByTAbvMtBN ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 12 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VQHZRNl7WQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.TEm04ElY8x ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VQHZRNl7WQ ++ cat /tmp/tmp.TEm04ElY8x ++ rm /tmp/tmp.VQHZRNl7WQ /tmp/tmp.TEm04ElY8x ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 13 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + grep NULL + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3DuPOz0lwu +++ mktemp ++ local LAST_ERR=/tmp/tmp.LpVNtcV4GX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3DuPOz0lwu ++ cat /tmp/tmp.LpVNtcV4GX ++ rm /tmp/tmp.3DuPOz0lwu /tmp/tmp.LpVNtcV4GX ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 14 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + grep NULL + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Kk0aJgtbJl +++ mktemp ++ local LAST_ERR=/tmp/tmp.ulUaZlaEji ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Kk0aJgtbJl ++ cat /tmp/tmp.ulUaZlaEji ++ rm /tmp/tmp.Kk0aJgtbJl /tmp/tmp.ulUaZlaEji ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + echo 'waiting for password propagation' waiting for password propagation + sleep 1 + let retry+=1 + [[ 15 -ge 240 ]] + is_old_password_discarded monitor '-h some-name-pxc -uroot -p'\''test-password'\''' + local username=monitor + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' + run_mysql 'SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' '-h some-name-pxc -uroot -p'\''test-password'\''' + local 'command=SELECT User_attributes FROM mysql.user WHERE user='\''monitor'\''' + grep NULL + local 'uri=-h some-name-pxc -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uBPxep6gbd +++ mktemp ++ local LAST_ERR=/tmp/tmp.3T45AX5r9o ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uBPxep6gbd ++ cat /tmp/tmp.3T45AX5r9o ++ rm /tmp/tmp.uBPxep6gbd /tmp/tmp.3T45AX5r9o ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace NULL + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.upP8SiRTT7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.cavIDGAZL5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.upP8SiRTT7 ++ cat /tmp/tmp.cavIDGAZL5 ++ rm /tmp/tmp.upP8SiRTT7 /tmp/tmp.cavIDGAZL5 ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4B0h0eueIC +++ mktemp ++ local LAST_ERR=/tmp/tmp.b68AXoV19l ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4B0h0eueIC ++ cat /tmp/tmp.b68AXoV19l ++ rm /tmp/tmp.4B0h0eueIC /tmp/tmp.b68AXoV19l ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.C3TJrQn3V3 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Glb3pSOTSk +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.C3TJrQn3V3 +++++ cat /tmp/tmp.Glb3pSOTSk +++++ rm /tmp/tmp.C3TJrQn3V3 /tmp/tmp.Glb3pSOTSk +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.FlgKozd99G ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.C9MYDduJJI +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.FlgKozd99G +++++ cat /tmp/tmp.C9MYDduJJI +++++ rm /tmp/tmp.FlgKozd99G /tmp/tmp.C9MYDduJJI +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.o0mCGup8f8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.V9eCxi7RMH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.o0mCGup8f8 ++ cat /tmp/tmp.V9eCxi7RMH ++ rm /tmp/tmp.o0mCGup8f8 /tmp/tmp.V9eCxi7RMH ++ return 0 + [[ 2 == \2 ]] + sleep 10 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql ]] + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7iVwNWUAD5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.YXKPTIr2HV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7iVwNWUAD5 ++ cat /tmp/tmp.YXKPTIr2HV ++ rm /tmp/tmp.7iVwNWUAD5 /tmp/tmp.YXKPTIr2HV ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.HcH8ZcrDqq/select-4.sql + desc 'test operator' + set +o xtrace ----------------------------------------------------------------------------------- test operator ----------------------------------------------------------------------------------- + patch_secret my-cluster-secrets operator dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=operator + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.7MNbfu5GYS ++ mktemp + local LAST_ERR=/tmp/tmp.ANGzrjKpCj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7MNbfu5GYS secret/my-cluster-secrets patched + cat /tmp/tmp.ANGzrjKpCj + rm /tmp/tmp.7MNbfu5GYS /tmp/tmp.ANGzrjKpCj + return 0 + sleep 15 + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qEGUU5T3kh +++ mktemp ++ local LAST_ERR=/tmp/tmp.aCytXqlyTv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qEGUU5T3kh ++ cat /tmp/tmp.aCytXqlyTv ++ rm /tmp/tmp.qEGUU5T3kh /tmp/tmp.aCytXqlyTv ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JQ5cr05bov +++ mktemp ++ local LAST_ERR=/tmp/tmp.0HTR1Lfc1N ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JQ5cr05bov ++ cat /tmp/tmp.0HTR1Lfc1N ++ rm /tmp/tmp.JQ5cr05bov /tmp/tmp.0HTR1Lfc1N ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.uH8uBVuCrN ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.A5AzaWoA7I +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.uH8uBVuCrN +++++ cat /tmp/tmp.A5AzaWoA7I +++++ rm /tmp/tmp.uH8uBVuCrN /tmp/tmp.A5AzaWoA7I +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.FaAAYcsdhJ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.JanzIbc6In +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.FaAAYcsdhJ +++++ cat /tmp/tmp.JanzIbc6In +++++ rm /tmp/tmp.FaAAYcsdhJ /tmp/tmp.JanzIbc6In +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SbzGKRB8Wy +++ mktemp ++ local LAST_ERR=/tmp/tmp.MraR3o9IdC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SbzGKRB8Wy ++ cat /tmp/tmp.MraR3o9IdC ++ rm /tmp/tmp.SbzGKRB8Wy /tmp/tmp.MraR3o9IdC ++ return 0 + [[ 2 == \2 ]] + sleep 10 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql ]] + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0yXEiRTFSs +++ mktemp ++ local LAST_ERR=/tmp/tmp.XATW07MQ2l ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0yXEiRTFSs ++ cat /tmp/tmp.XATW07MQ2l ++ rm /tmp/tmp.0yXEiRTFSs /tmp/tmp.XATW07MQ2l ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.HcH8ZcrDqq/select-4.sql + desc 'change secret name' + set +o xtrace ----------------------------------------------------------------------------------- change secret name ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.k5WNRoj9nr ++ mktemp + local LAST_ERR=/tmp/tmp.waTmHNiwfH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.k5WNRoj9nr perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.waTmHNiwfH + rm /tmp/tmp.k5WNRoj9nr /tmp/tmp.waTmHNiwfH + return 0 + sleep 30 + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gOcONFZ2rI +++ mktemp ++ local LAST_ERR=/tmp/tmp.w6cdCdqlZL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gOcONFZ2rI ++ cat /tmp/tmp.w6cdCdqlZL ++ rm /tmp/tmp.gOcONFZ2rI /tmp/tmp.w6cdCdqlZL ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VgfmMp2ikK +++ mktemp ++ local LAST_ERR=/tmp/tmp.zUEYHVAi9o ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VgfmMp2ikK ++ cat /tmp/tmp.zUEYHVAi9o ++ rm /tmp/tmp.VgfmMp2ikK /tmp/tmp.zUEYHVAi9o ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dA0tVIy4ns +++ mktemp ++ local LAST_ERR=/tmp/tmp.InkXFb7q0P ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dA0tVIy4ns ++ cat /tmp/tmp.InkXFb7q0P ++ rm /tmp/tmp.dA0tVIy4ns /tmp/tmp.InkXFb7q0P ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.75hBT0azNA +++ mktemp ++ local LAST_ERR=/tmp/tmp.sFGsV7pUR4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.75hBT0azNA ++ cat /tmp/tmp.sFGsV7pUR4 ++ rm /tmp/tmp.75hBT0azNA /tmp/tmp.sFGsV7pUR4 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Bc4iN14H3F +++ mktemp ++ local LAST_ERR=/tmp/tmp.EHXEmKeMTh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Bc4iN14H3F ++ cat /tmp/tmp.EHXEmKeMTh ++ rm /tmp/tmp.Bc4iN14H3F /tmp/tmp.EHXEmKeMTh ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3kCF1WFXMo +++ mktemp ++ local LAST_ERR=/tmp/tmp.iP3SBvm4bv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3kCF1WFXMo ++ cat /tmp/tmp.iP3SBvm4bv ++ rm /tmp/tmp.3kCF1WFXMo /tmp/tmp.iP3SBvm4bv ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KBBj5Q2JS1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.pqWDIdrvA3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KBBj5Q2JS1 ++ cat /tmp/tmp.pqWDIdrvA3 ++ rm /tmp/tmp.KBBj5Q2JS1 /tmp/tmp.pqWDIdrvA3 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4gLNkAf6B9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.8sw5fUfInU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4gLNkAf6B9 ++ cat /tmp/tmp.8sw5fUfInU ++ rm /tmp/tmp.4gLNkAf6B9 /tmp/tmp.8sw5fUfInU ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.s3nwLGabT4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.iegpiUx7PA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.s3nwLGabT4 ++ cat /tmp/tmp.iegpiUx7PA ++ rm /tmp/tmp.s3nwLGabT4 /tmp/tmp.iegpiUx7PA ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ff3sO0qrnb ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.UeYW6K2gc0 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ff3sO0qrnb +++++ cat /tmp/tmp.UeYW6K2gc0 +++++ rm /tmp/tmp.ff3sO0qrnb /tmp/tmp.UeYW6K2gc0 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.PnI6FW2eR5 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.UifkMonBMO +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.PnI6FW2eR5 +++++ cat /tmp/tmp.UifkMonBMO +++++ rm /tmp/tmp.PnI6FW2eR5 /tmp/tmp.UifkMonBMO +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bIKsj3Shbo +++ mktemp ++ local LAST_ERR=/tmp/tmp.HnCTyl8AmD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bIKsj3Shbo ++ cat /tmp/tmp.HnCTyl8AmD ++ rm /tmp/tmp.bIKsj3Shbo /tmp/tmp.HnCTyl8AmD ++ return 0 + [[ 2 == \2 ]] + desc 'test new operator' + set +o xtrace ----------------------------------------------------------------------------------- test new operator ----------------------------------------------------------------------------------- + newpass=test-password2 ++ echo -n test-password2 ++ base64 + newpassencrypted=dGVzdC1wYXNzd29yZDI= + patch_secret my-cluster-secrets-2 operator dGVzdC1wYXNzd29yZDI= + local secret=my-cluster-secrets-2 + local key=operator + local value=dGVzdC1wYXNzd29yZDI= + kubectl_bin patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.WwzN1MXJzW ++ mktemp + local LAST_ERR=/tmp/tmp.uNVWcS0h5n + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WwzN1MXJzW secret/my-cluster-secrets-2 patched + cat /tmp/tmp.uNVWcS0h5n + rm /tmp/tmp.WwzN1MXJzW /tmp/tmp.uNVWcS0h5n + return 0 + sleep 15 + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KQGjNSCUjK +++ mktemp ++ local LAST_ERR=/tmp/tmp.IOqB9uR4Ip ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KQGjNSCUjK ++ cat /tmp/tmp.IOqB9uR4Ip ++ rm /tmp/tmp.KQGjNSCUjK /tmp/tmp.IOqB9uR4Ip ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KHJkOO50nY +++ mktemp ++ local LAST_ERR=/tmp/tmp.QvRX54e3Qs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KHJkOO50nY ++ cat /tmp/tmp.QvRX54e3Qs ++ rm /tmp/tmp.KHJkOO50nY /tmp/tmp.QvRX54e3Qs ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8nSDeKd38T +++ mktemp ++ local LAST_ERR=/tmp/tmp.UPvciq6gRm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8nSDeKd38T ++ cat /tmp/tmp.UPvciq6gRm ++ rm /tmp/tmp.8nSDeKd38T /tmp/tmp.UPvciq6gRm ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.BGCkxM85vM ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.QSRxt4f6ew +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.BGCkxM85vM +++++ cat /tmp/tmp.QSRxt4f6ew +++++ rm /tmp/tmp.BGCkxM85vM /tmp/tmp.QSRxt4f6ew +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.2539ElmJ8g ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.i8yLK0NdG0 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.2539ElmJ8g +++++ cat /tmp/tmp.i8yLK0NdG0 +++++ rm /tmp/tmp.2539ElmJ8g /tmp/tmp.i8yLK0NdG0 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.87piLiS0aa +++ mktemp ++ local LAST_ERR=/tmp/tmp.uGe0w0b1i7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.87piLiS0aa ++ cat /tmp/tmp.uGe0w0b1i7 ++ rm /tmp/tmp.87piLiS0aa /tmp/tmp.uGe0w0b1i7 ++ return 0 + [[ 2 == \2 ]] + sleep 20 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql ]] + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.buOX5GS8Hi +++ mktemp ++ local LAST_ERR=/tmp/tmp.4Akrh3tyFf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.buOX5GS8Hi ++ cat /tmp/tmp.4Akrh3tyFf ++ rm /tmp/tmp.buOX5GS8Hi /tmp/tmp.4Akrh3tyFf ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.HcH8ZcrDqq/select-4.sql ++ getSecretData my-cluster-secrets-2 root ++ local secretName=my-cluster-secrets-2 ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets-2 '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.ADtVm53J8t +++ mktemp ++ local LAST_ERR=/tmp/tmp.lcWYpr4pIJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets-2 '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ADtVm53J8t ++ cat /tmp/tmp.lcWYpr4pIJ ++ rm /tmp/tmp.ADtVm53J8t /tmp/tmp.lcWYpr4pIJ ++ return 0 + newpass='CN],Rw,ciotS*v,f&v' + desc 'test new users sync' + set +o xtrace ----------------------------------------------------------------------------------- test new users sync ----------------------------------------------------------------------------------- + run_mysql 'CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\''CN],Rw,ciotS*v,f&v'\'';' '-h some-name-pxc -uroot -p'\''CN],Rw,ciotS*v,f&v'\''' + local 'command=CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\''CN],Rw,ciotS*v,f&v'\'';' + local 'uri=-h some-name-pxc -uroot -p'\''CN],Rw,ciotS*v,f&v'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3jNKYFeh0u +++ mktemp ++ local LAST_ERR=/tmp/tmp.BlgpcXgnbb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3jNKYFeh0u ++ cat /tmp/tmp.BlgpcXgnbb ++ rm /tmp/tmp.3jNKYFeh0u /tmp/tmp.BlgpcXgnbb ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + sleep 40 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\''CN],Rw,ciotS*v,f&v'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -utestsync -p'\''CN],Rw,ciotS*v,f&v'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql ]] + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\''CN],Rw,ciotS*v,f&v'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -utestsync -p'\''CN],Rw,ciotS*v,f&v'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TOWVTdSQrR +++ mktemp ++ local LAST_ERR=/tmp/tmp.LksHnIVX4g ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TOWVTdSQrR ++ cat /tmp/tmp.LksHnIVX4g ++ rm /tmp/tmp.TOWVTdSQrR /tmp/tmp.LksHnIVX4g ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.HcH8ZcrDqq/select-4.sql ++ getSecretData internal-some-name operator ++ local secretName=internal-some-name ++ local dataKey=operator ++ kubectl_bin get secrets/internal-some-name '--template={{.data.operator}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.EkWZmV7DO4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.IBArDxmtE9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/internal-some-name '--template={{.data.operator}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EkWZmV7DO4 ++ cat /tmp/tmp.IBArDxmtE9 ++ rm /tmp/tmp.EkWZmV7DO4 /tmp/tmp.IBArDxmtE9 ++ return 0 + pass=test-password2 + desc 'check secret without operator' + set +o xtrace ----------------------------------------------------------------------------------- check secret without operator ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.5iKgJxTMPY ++ mktemp + local LAST_ERR=/tmp/tmp.Igk6hpL7k5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5iKgJxTMPY secret/my-cluster-secrets-2 configured + cat /tmp/tmp.Igk6hpL7k5 Warning: resource secrets/my-cluster-secrets-2 is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.5iKgJxTMPY /tmp/tmp.Igk6hpL7k5 + return 0 + sleep 15 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql ]] + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.71iUoxW7Ir +++ mktemp ++ local LAST_ERR=/tmp/tmp.TG7bD2ziIj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.71iUoxW7Ir ++ cat /tmp/tmp.TG7bD2ziIj ++ rm /tmp/tmp.71iUoxW7Ir /tmp/tmp.TG7bD2ziIj ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-4-80.sql /tmp/tmp.HcH8ZcrDqq/select-4.sql + newpass=test-password2 ++ echo -n test-password2 ++ base64 + newpassencrypted=dGVzdC1wYXNzd29yZDI= + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/conf/some-name.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/conf/some-name.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/conf/some-name.yml ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.zMWS7FE4j7 + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1948-70c11cfc#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-729~ + local LAST_ERR=/tmp/tmp.dDSMnk4f3x + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zMWS7FE4j7 perconaxtradbcluster.pxc.percona.com/some-name configured + cat /tmp/tmp.dDSMnk4f3x + rm /tmp/tmp.zMWS7FE4j7 /tmp/tmp.dDSMnk4f3x + return 0 + sleep 15 + wait_cluster_consistency some-name 3 3 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=3 + '[' -z 3 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RGluPCheCE +++ mktemp ++ local LAST_ERR=/tmp/tmp.psQ0J8NFy5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RGluPCheCE ++ cat /tmp/tmp.psQ0J8NFy5 ++ rm /tmp/tmp.RGluPCheCE /tmp/tmp.psQ0J8NFy5 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.R6uFVhJq5Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.SlUVTwRn82 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.R6uFVhJq5Z ++ cat /tmp/tmp.SlUVTwRn82 ++ rm /tmp/tmp.R6uFVhJq5Z /tmp/tmp.SlUVTwRn82 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nU7Gkycbcc +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ij0cVxU2l1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nU7Gkycbcc ++ cat /tmp/tmp.Ij0cVxU2l1 ++ rm /tmp/tmp.nU7Gkycbcc /tmp/tmp.Ij0cVxU2l1 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wvxGsKawG5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.jj8q9fwO1K ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wvxGsKawG5 ++ cat /tmp/tmp.jj8q9fwO1K ++ rm /tmp/tmp.wvxGsKawG5 /tmp/tmp.jj8q9fwO1K ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yiHO255sWn +++ mktemp ++ local LAST_ERR=/tmp/tmp.Lg5AJKQGjE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yiHO255sWn ++ cat /tmp/tmp.Lg5AJKQGjE ++ rm /tmp/tmp.yiHO255sWn /tmp/tmp.Lg5AJKQGjE ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.F7GFumDqQs +++ mktemp ++ local LAST_ERR=/tmp/tmp.MSJ0ftvQSS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.F7GFumDqQs ++ cat /tmp/tmp.MSJ0ftvQSS ++ rm /tmp/tmp.F7GFumDqQs /tmp/tmp.MSJ0ftvQSS ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Yyi13arDFs +++ mktemp ++ local LAST_ERR=/tmp/tmp.OQ4OXWUoDK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Yyi13arDFs ++ cat /tmp/tmp.OQ4OXWUoDK ++ rm /tmp/tmp.Yyi13arDFs /tmp/tmp.OQ4OXWUoDK ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OdGdcVYd5c +++ mktemp ++ local LAST_ERR=/tmp/tmp.PQsDBldrs9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OdGdcVYd5c ++ cat /tmp/tmp.PQsDBldrs9 ++ rm /tmp/tmp.OdGdcVYd5c /tmp/tmp.PQsDBldrs9 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CaCdBbum0p +++ mktemp ++ local LAST_ERR=/tmp/tmp.MuBXUMJ1FC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CaCdBbum0p ++ cat /tmp/tmp.MuBXUMJ1FC ++ rm /tmp/tmp.CaCdBbum0p /tmp/tmp.MuBXUMJ1FC ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 8 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MIw9Xjc0FD +++ mktemp ++ local LAST_ERR=/tmp/tmp.DTd296hvzP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MIw9Xjc0FD ++ cat /tmp/tmp.DTd296hvzP ++ rm /tmp/tmp.MIw9Xjc0FD /tmp/tmp.DTd296hvzP ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 9 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kzxUyedEAW +++ mktemp ++ local LAST_ERR=/tmp/tmp.MWG7JSHYCU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kzxUyedEAW ++ cat /tmp/tmp.MWG7JSHYCU ++ rm /tmp/tmp.kzxUyedEAW /tmp/tmp.MWG7JSHYCU ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 10 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U97IO3W1IY +++ mktemp ++ local LAST_ERR=/tmp/tmp.fwWJYOnXwq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.U97IO3W1IY ++ cat /tmp/tmp.fwWJYOnXwq ++ rm /tmp/tmp.U97IO3W1IY /tmp/tmp.fwWJYOnXwq ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 11 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.44NHnQWT0S +++ mktemp ++ local LAST_ERR=/tmp/tmp.qaykit7qjc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.44NHnQWT0S ++ cat /tmp/tmp.qaykit7qjc ++ rm /tmp/tmp.44NHnQWT0S /tmp/tmp.qaykit7qjc ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jMBVOBwk1b +++ mktemp ++ local LAST_ERR=/tmp/tmp.1sK4modznJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jMBVOBwk1b ++ cat /tmp/tmp.1sK4modznJ ++ rm /tmp/tmp.jMBVOBwk1b /tmp/tmp.1sK4modznJ ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Vt0UOFNlEC ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.VvwWj32CFh +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Vt0UOFNlEC +++++ cat /tmp/tmp.VvwWj32CFh +++++ rm /tmp/tmp.Vt0UOFNlEC /tmp/tmp.VvwWj32CFh +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-haproxy ++++ return +++ local cluster_proxy=some-name-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jBvxdma6rW +++ mktemp ++ local LAST_ERR=/tmp/tmp.TvmRwXnvcc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jBvxdma6rW ++ cat /tmp/tmp.TvmRwXnvcc ++ rm /tmp/tmp.jBvxdma6rW /tmp/tmp.TvmRwXnvcc ++ return 0 + [[ 3 == \3 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + check_generation 1 haproxy some-name + local generation=1 + local container=haproxy + local cluster=some-name + local current_generation ++ kubectl_bin get statefulset some-name-haproxy -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YZ3moDzkSX +++ mktemp ++ local LAST_ERR=/tmp/tmp.vRXGFrz9Tc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get statefulset some-name-haproxy -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YZ3moDzkSX ++ cat /tmp/tmp.vRXGFrz9Tc ++ rm /tmp/tmp.YZ3moDzkSX /tmp/tmp.vRXGFrz9Tc ++ return 0 + current_generation=1 + [[ 1 != \1 ]] + patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZDI= + local secret=my-cluster-secrets + local key=monitor + local value=dGVzdC1wYXNzd29yZDI= + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.35Z6mytPdK ++ mktemp + local LAST_ERR=/tmp/tmp.piBqmL1SV2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.35Z6mytPdK secret/my-cluster-secrets patched + cat /tmp/tmp.piBqmL1SV2 + rm /tmp/tmp.35Z6mytPdK /tmp/tmp.piBqmL1SV2 + return 0 + sleep 15 + wait_cluster_consistency some-name 3 3 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=3 + '[' -z 3 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ij3qp5uLn4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.R6XUDG62BY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ij3qp5uLn4 ++ cat /tmp/tmp.R6XUDG62BY ++ rm /tmp/tmp.ij3qp5uLn4 /tmp/tmp.R6XUDG62BY ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rVCWDs5jC7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.oH2c9EQVLt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rVCWDs5jC7 ++ cat /tmp/tmp.oH2c9EQVLt ++ rm /tmp/tmp.rVCWDs5jC7 /tmp/tmp.oH2c9EQVLt ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.EEdS6sNWar ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.jCzFYVyh9M +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.EEdS6sNWar +++++ cat /tmp/tmp.jCzFYVyh9M +++++ rm /tmp/tmp.EEdS6sNWar /tmp/tmp.jCzFYVyh9M +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-haproxy ++++ return +++ local cluster_proxy=some-name-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hPtlPKdlhO +++ mktemp ++ local LAST_ERR=/tmp/tmp.UysRkIcu0L ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hPtlPKdlhO ++ cat /tmp/tmp.UysRkIcu0L ++ rm /tmp/tmp.hPtlPKdlhO /tmp/tmp.UysRkIcu0L ++ return 0 + [[ 3 == \3 ]] + compare_mysql_cmd select-3 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\''' + local command_id=select-3 + local 'command=SHOW DATABASES;' + local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-3-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\''' + local 'command=SHOW DATABASES;' + local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VCnbCNUn6L +++ mktemp ++ local LAST_ERR=/tmp/tmp.AJvWs5ocY9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VCnbCNUn6L ++ cat /tmp/tmp.AJvWs5ocY9 ++ rm /tmp/tmp.VCnbCNUn6L /tmp/tmp.AJvWs5ocY9 ++ return 0 + client_pod=pxc-client-69f9bd646c-g2t2l + wait_pod pxc-client-69f9bd646c-g2t2l + local pod=pxc-client-69f9bd646c-g2t2l + local max_retry=480 + local ns= ++ echo pxc-client-69f9bd646c-g2t2l ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-69f9bd646c-g2t2l condition met waiting for pod/pxc-client-69f9bd646c-g2t2l to become Ready.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.HcH8ZcrDqq/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1948/e2e-tests/users/compare/select-3.sql /tmp/tmp.HcH8ZcrDqq/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + check_generation 1 haproxy some-name + local generation=1 + local container=haproxy + local cluster=some-name + local current_generation ++ kubectl_bin get statefulset some-name-haproxy -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zsaKDGKC1b +++ mktemp ++ local LAST_ERR=/tmp/tmp.UyboXKc8uB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get statefulset some-name-haproxy -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zsaKDGKC1b ++ cat /tmp/tmp.UyboXKc8uB ++ rm /tmp/tmp.zsaKDGKC1b /tmp/tmp.UyboXKc8uB ++ return 0 + current_generation=1 + [[ 1 != \1 ]] + destroy users-729 + local namespace=users-729 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' ++ get_operator_pod + grep -v 'the object has been modified' ++ local label_prefix=app.kubernetes.io/ + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + sort -u + tee /tmp/tmp.HcH8ZcrDqq/operator.log +++ grep -c percona-xtradb-cluster-operator + grep -v 'get backup status: Job.batch' + grep -v level=info +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.7a9JWDb4sf +++ mktemp ++ local LAST_ERR=/tmp/tmp.1JIMGocLRR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7a9JWDb4sf ++ cat /tmp/tmp.1JIMGocLRR ++ rm /tmp/tmp.7a9JWDb4sf /tmp/tmp.1JIMGocLRR ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-5f5f9cbb4-mq4gx ++ mktemp + local LAST_OUT=/tmp/tmp.PKOADkTcga ++ mktemp + local LAST_ERR=/tmp/tmp.TGLMVtxuNx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-5f5f9cbb4-mq4gx + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PKOADkTcga + cat /tmp/tmp.TGLMVtxuNx + rm /tmp/tmp.PKOADkTcga /tmp/tmp.TGLMVtxuNx + return 0 2025-01-22T14:20:07.792Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.28.15-gke.1612000"} 2025-01-22T14:20:07.793Z INFO setup Manager starting up {"gitCommit": "70c11cfc9edb0f5dee7a6f18a91120ac58f6385e", "gitBranch": "PR-1948-70c11cfc", "buildTime": "2025-01-22T12:05:11Z", "goVersion": "go1.23.5", "os": "linux", "arch": "amd64"} 2025-01-22T14:20:07.794Z INFO setup Registering Components. 2025-01-22T14:20:11.785Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2025-01-22T14:20:11.788Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2025-01-22T14:20:11.788Z INFO controller-runtime.metrics Starting metrics server 2025-01-22T14:20:11.788Z INFO setup Starting the Cmd. 2025-01-22T14:20:11.788Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2025-01-22T14:20:11.789Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"interval": "10s"} 2025-01-22T14:20:11.789Z INFO controller-runtime.certwatcher Updated current TLS certificate 2025-01-22T14:20:11.789Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2025-01-22T14:20:11.789Z INFO controller-runtime.webhook Starting webhook server 2025-01-22T14:20:11.990Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2025-01-22T14:20:29.218Z DEBUG events percona-xtradb-cluster-operator-5f5f9cbb4-mq4gx_e3f8bae5-2bbe-4a2f-911d-ab4eedae9b9d became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"7fd8806d-2036-47a9-bd97-a7773090f621","apiVersion":"coordination.k8s.io/v1","resourceVersion":"84180"}, "reason": "LeaderElection"} 2025-01-22T14:20:29.218Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2025-01-22T14:20:29.218Z INFO Starting Controller {"controller": "pxc-controller"} 2025-01-22T14:20:29.218Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2025-01-22T14:20:29.218Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2025-01-22T14:20:29.218Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"} 2025-01-22T14:20:29.218Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2025-01-22T14:20:29.218Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2025-01-22T14:20:29.321Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2025-01-22T14:20:29.321Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2025-01-22T14:20:29.321Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2025-01-22T14:21:16.861Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "93eee7a8-e102-46a3-9dfb-33fb8ed67657", "err": "get primary pxc pod: failed to get proxy connection: dial tcp: lookup some-name-proxysql-unready.users-729: i/o timeout"} 2025-01-22T14:28:49.364Z INFO We are in full cluster crash, starting recovery {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "6ebb7ff6-1f9e-4526-b78a-4805b440b93f"} 2025-01-22T14:28:49.365Z INFO Results of scanning sequences {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "6ebb7ff6-1f9e-4526-b78a-4805b440b93f", "pod": "some-name-pxc-0", "maxSeq": 47} 2025-01-22T14:29:19.903Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "76fec5a4-ae1a-48be-b2c9-13235b4ee232", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-2: dial tcp 10.231.2.3:33062: connect: connection refused"} 2025-01-22T14:29:24.882Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "09d91c4c-855c-4844-9653-179837709ee3", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp 10.231.1.57:33062: connect: connection refused"} 2025-01-22T14:29:30.063Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "848135b8-df1e-416a-9f5e-5b9130c03140", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp 10.231.1.57:33062: connect: connection refused"} 2025-01-22T14:29:35.312Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "93739c1a-7b0d-4cb0-a2f9-cd3531a554da", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-2: dial tcp 10.231.2.3:33062: connect: connection refused"} 2025-01-22T14:29:40.486Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "e33ba757-29ad-4dcb-a7a6-5c021e548e86", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp 10.231.1.57:33062: connect: connection refused"} 2025-01-22T14:30:34.502Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "2f615600-70dd-471e-bb22-ad9079b7bc1f"} 2025-01-22T14:30:39.804Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "a9fd8bac-6c8d-4e53-b469-5910d1cf4b7f"} 2025-01-22T14:30:45.283Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "478cab8c-729e-4063-b699-8a459311063b"} 2025-01-22T14:30:47.642Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "2ad2b429-2085-417f-a4b0-4d26d2c91b8b", "user": "monitor"} 2025-01-22T14:30:47.661Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "2ad2b429-2085-417f-a4b0-4d26d2c91b8b", "user": "monitor"} 2025-01-22T14:30:47.670Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "2ad2b429-2085-417f-a4b0-4d26d2c91b8b", "secret": "some-name-mysql-init", "user": "monitor"} 2025-01-22T14:30:47.703Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "2ad2b429-2085-417f-a4b0-4d26d2c91b8b", "user": "monitor"} 2025-01-22T14:30:47.710Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "2ad2b429-2085-417f-a4b0-4d26d2c91b8b", "user": "monitor"} 2025-01-22T14:30:48.007Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "2ad2b429-2085-417f-a4b0-4d26d2c91b8b", "last-applied-secret": "4e3282050745046d5044e0fcb5f2bf140d7320a1256a79c8ca2184f38f4f3098"} 2025-01-22T14:30:48.017Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "2ad2b429-2085-417f-a4b0-4d26d2c91b8b", "object": "some-name-proxysql"} 2025-01-22T14:30:50.342Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "2ad2b429-2085-417f-a4b0-4d26d2c91b8b", "error": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") / / ", "errorVerbose": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") / / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:925\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} 2025-01-22T14:32:01.089Z INFO Password updated but old one not discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "66b9ef41-69d7-43dd-a2e5-15deb19658c7", "user": "monitor"} 2025-01-22T14:32:01.926Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "66b9ef41-69d7-43dd-a2e5-15deb19658c7", "user": "monitor"} 2025-01-22T14:32:01.945Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "66b9ef41-69d7-43dd-a2e5-15deb19658c7", "last-applied-secret": "4e3282050745046d5044e0fcb5f2bf140d7320a1256a79c8ca2184f38f4f3098"} 2025-01-22T14:32:04.871Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "66b9ef41-69d7-43dd-a2e5-15deb19658c7"} 2025-01-22T14:32:08.723Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "8d1bbe11-665c-4129-a692-31b9dea79ece"} 2025-01-22T14:32:14.323Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "67ac3935-3ef4-450f-b8d7-5990bdfffbe0"} 2025-01-22T14:32:19.584Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f44a5341-8a3c-488d-b163-d04953867170"} 2025-01-22T14:32:24.400Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "fd8f274c-c7a6-45f6-96e7-ddcb43d19198"} 2025-01-22T14:32:27.033Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f18d91bd-86b2-4413-82b3-734ca05c2865", "user": "operator"} 2025-01-22T14:32:27.055Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f18d91bd-86b2-4413-82b3-734ca05c2865", "user": "operator"} 2025-01-22T14:32:27.062Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f18d91bd-86b2-4413-82b3-734ca05c2865", "secret": "some-name-mysql-init", "user": "operator"} 2025-01-22T14:32:27.070Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f18d91bd-86b2-4413-82b3-734ca05c2865", "user": "operator"} 2025-01-22T14:32:27.093Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f18d91bd-86b2-4413-82b3-734ca05c2865", "user": "operator"} 2025-01-22T14:32:27.124Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f18d91bd-86b2-4413-82b3-734ca05c2865", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"} 2025-01-22T14:32:27.191Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f18d91bd-86b2-4413-82b3-734ca05c2865", "object": "some-name-proxysql"} 2025-01-22T14:32:32.002Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f18d91bd-86b2-4413-82b3-734ca05c2865", "error": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:925\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} 2025-01-22T14:32:51.596Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "42aa26b7-8077-4d27-86b6-f4ac9b322d53"} 2025-01-22T14:33:05.511Z INFO Created user secrets {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "secrets": "my-cluster-secrets-2"} 2025-01-22T14:33:05.519Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "root"} 2025-01-22T14:33:05.556Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "root"} 2025-01-22T14:33:05.563Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "secret": "some-name-mysql-init", "user": "root"} 2025-01-22T14:33:07.343Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "6e25afd4-e9af-4b40-a36c-94ab4acc30e0"} 2025-01-22T14:33:08.778Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20"} 2025-01-22T14:33:08.787Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "root"} 2025-01-22T14:33:08.826Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "root"} 2025-01-22T14:33:08.835Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "operator"} 2025-01-22T14:33:08.855Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "operator"} 2025-01-22T14:33:08.861Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "secret": "some-name-mysql-init", "user": "operator"} 2025-01-22T14:33:08.869Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "operator"} 2025-01-22T14:33:08.886Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "operator"} 2025-01-22T14:33:08.896Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "monitor"} 2025-01-22T14:33:08.914Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "monitor"} 2025-01-22T14:33:08.922Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "secret": "some-name-mysql-init", "user": "monitor"} 2025-01-22T14:33:08.954Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "monitor"} 2025-01-22T14:33:08.963Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "monitor"} 2025-01-22T14:33:09.241Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "xtrabackup"} 2025-01-22T14:33:09.263Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "xtrabackup"} 2025-01-22T14:33:09.271Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "secret": "some-name-mysql-init", "user": "xtrabackup"} 2025-01-22T14:33:09.279Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "xtrabackup"} 2025-01-22T14:33:09.295Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "xtrabackup"} 2025-01-22T14:33:09.305Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "replication"} 2025-01-22T14:33:09.320Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "replication"} 2025-01-22T14:33:09.327Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "secret": "some-name-mysql-init", "user": "replication"} 2025-01-22T14:33:09.336Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "replication"} 2025-01-22T14:33:09.358Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "replication"} 2025-01-22T14:33:09.358Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "proxyadmin"} 2025-01-22T14:33:09.395Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "proxyadmin"} 2025-01-22T14:33:09.435Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "user": "proxyadmin"} 2025-01-22T14:33:09.435Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "last-applied-secret": "b86a3b7c8116a20c99b9a6152ff7d97ed41289bcd3cab37ec358068f45efb9e4"} 2025-01-22T14:33:09.435Z INFO PXC pods will be restarted {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "last-applied-secret": "b86a3b7c8116a20c99b9a6152ff7d97ed41289bcd3cab37ec358068f45efb9e4"} 2025-01-22T14:33:09.441Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "object": "some-name-pxc"} 2025-01-22T14:33:09.689Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "object": "some-name-proxysql"} 2025-01-22T14:33:11.729Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e310df4-0d48-43c8-9f61-22e7d5ea3c20", "error": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:925\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} 2025-01-22T14:34:11.932Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "96bd87e9-a816-41f0-8929-6c0d701ce0b7", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-729 on 10.94.144.10:53: no such host"} 2025-01-22T14:34:17.053Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "c746401c-23f0-4a5f-843b-fef3b069ca9f", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-729 on 10.94.144.10:53: no such host"} 2025-01-22T14:34:53.196Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "518f6182-7e2f-4883-8574-767ac947515d", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-729 on 10.94.144.10:53: no such host"} 2025-01-22T14:34:58.204Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "5496274b-dff8-441d-8df5-d66ebf0359e4", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-729 on 10.94.144.10:53: no such host"} 2025-01-22T14:35:03.308Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "2c99f396-aba9-42dd-9431-3ca4850d39f4", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-729 on 10.94.144.10:53: no such host"} 2025-01-22T14:35:08.460Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "e86a41e7-ad23-4ab7-b619-205959c18945", "primary name": "some-name-pxc-0.some-name-pxc.users-729.svc.cluster.local"} 2025-01-22T14:35:13.519Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "20335720-bd8c-4022-8588-d889a76ce685", "primary name": "some-name-pxc-0.some-name-pxc.users-729.svc.cluster.local"} 2025-01-22T14:35:18.595Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "0815c2fd-16bc-4c3c-8332-274ffe8db39c", "primary name": "some-name-pxc-0.some-name-pxc.users-729.svc.cluster.local"} 2025-01-22T14:35:23.668Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "93cfc9c4-c6c5-4d2f-a685-57f7486a13dc", "primary name": "some-name-pxc-0.some-name-pxc.users-729.svc.cluster.local"} 2025-01-22T14:35:28.736Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f0ff7a2f-3d46-4004-b6a4-6cfcd14e73e5", "primary name": "some-name-pxc-0.some-name-pxc.users-729.svc.cluster.local"} 2025-01-22T14:35:49.374Z INFO Password updated but old one not discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "862b2b4f-d350-4728-b3f9-73068dad329b", "user": "monitor"} 2025-01-22T14:35:50.020Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "862b2b4f-d350-4728-b3f9-73068dad329b", "user": "monitor"} 2025-01-22T14:35:50.039Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "862b2b4f-d350-4728-b3f9-73068dad329b", "last-applied-secret": "b86a3b7c8116a20c99b9a6152ff7d97ed41289bcd3cab37ec358068f45efb9e4"} 2025-01-22T14:35:53.011Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "862b2b4f-d350-4728-b3f9-73068dad329b"} 2025-01-22T14:35:57.402Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "5cbb7fdf-d698-48ec-8980-a8fdf2fe40c1"} 2025-01-22T14:36:02.491Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d02d10fe-8052-4f0c-8595-440ed6af1edf"} 2025-01-22T14:36:07.807Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "86259c6d-91f3-4d32-befa-926e3e23dcf1"} 2025-01-22T14:36:10.411Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "bf2e3258-caa4-4c49-99a0-7541669f5a98", "user": "operator"} 2025-01-22T14:36:10.435Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "bf2e3258-caa4-4c49-99a0-7541669f5a98", "user": "operator"} 2025-01-22T14:36:10.441Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "bf2e3258-caa4-4c49-99a0-7541669f5a98", "secret": "some-name-mysql-init", "user": "operator"} 2025-01-22T14:36:10.450Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "bf2e3258-caa4-4c49-99a0-7541669f5a98", "user": "operator"} 2025-01-22T14:36:10.468Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "bf2e3258-caa4-4c49-99a0-7541669f5a98", "user": "operator"} 2025-01-22T14:36:10.501Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "bf2e3258-caa4-4c49-99a0-7541669f5a98", "last-applied-secret": "ee0db5fcc6946796c3e7add96b4f280df058ec11df4f9c71385bc645fee00f06"} 2025-01-22T14:36:10.510Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "bf2e3258-caa4-4c49-99a0-7541669f5a98", "object": "some-name-proxysql"} 2025-01-22T14:36:15.536Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "bf2e3258-caa4-4c49-99a0-7541669f5a98", "error": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-729.svc.cl' (using password: YES)\nERROR (line:572) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:925\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} 2025-01-22T14:36:39.026Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1e4c78ef-9e3a-45f1-899c-fb4a33bd3722"} 2025-01-22T14:36:54.551Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "9de769f0-e44d-425d-a72e-f0fc23dee4fd"} 2025-01-22T14:36:59.212Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "5614bd5d-82ea-4dde-b205-844b063e829d"} 2025-01-22T14:37:04.419Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "0c3121d6-6459-430d-8062-2fa6339b7c59"} 2025-01-22T14:37:09.971Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "c5922777-c345-4dd6-bb19-67271608bbe6"} 2025-01-22T14:37:15.320Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f6d56713-ecd5-4198-a351-a07fe38e160f"} 2025-01-22T14:37:20.679Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "18d101c5-3744-43e7-add6-7a8bc13e6450"} 2025-01-22T14:37:26.538Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f689b64d-f1af-4c56-a7d5-77cb9cc5a651"} 2025-01-22T14:37:30.984Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4853db9e-86dc-47e8-a15d-196fb2c2d674"} 2025-01-22T14:37:36.527Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "50dc9a6b-0e21-46e9-aa4c-993ddf476ec7"} 2025-01-22T14:37:41.687Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "c03cfd81-aca6-481f-847f-bbfd03e8553b"} 2025-01-22T14:37:47.104Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "c2424d01-3d1e-4e20-be9a-c8c988d02598"} 2025-01-22T14:37:52.203Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "e7e67397-5f32-4474-96af-4adc4cef1177"} 2025-01-22T14:37:57.721Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "fef83067-4680-4367-a5f3-fcae6e821c77"} 2025-01-22T14:38:03.125Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "a1f0f387-3402-4c1b-a030-1a9fc643e2af"} 2025-01-22T14:38:08.200Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "733b0311-a04d-4219-adff-435a3d5d39ee"} 2025-01-22T14:38:10.727Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "root"} 2025-01-22T14:38:10.763Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "root"} 2025-01-22T14:38:10.771Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "secret": "some-name-mysql-init", "user": "root"} 2025-01-22T14:38:14.514Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d"} 2025-01-22T14:38:14.523Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "root"} 2025-01-22T14:38:14.563Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "root"} 2025-01-22T14:38:14.589Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "monitor"} 2025-01-22T14:38:14.613Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "monitor"} 2025-01-22T14:38:14.619Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "secret": "some-name-mysql-init", "user": "monitor"} 2025-01-22T14:38:14.656Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "monitor"} 2025-01-22T14:38:14.665Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "monitor"} 2025-01-22T14:38:14.951Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "xtrabackup"} 2025-01-22T14:38:14.982Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "xtrabackup"} 2025-01-22T14:38:14.990Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "secret": "some-name-mysql-init", "user": "xtrabackup"} 2025-01-22T14:38:14.997Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "xtrabackup"} 2025-01-22T14:38:15.022Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "xtrabackup"} 2025-01-22T14:38:15.032Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "proxyadmin"} 2025-01-22T14:38:15.067Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "proxyadmin"} 2025-01-22T14:38:15.075Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "user": "proxyadmin"} 2025-01-22T14:38:15.075Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "last-applied-secret": "5e44c645bc123d23c86ea79cc836cb854ea07cac5341bafade616e18ec276303"} 2025-01-22T14:38:15.075Z INFO PXC pods will be restarted {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "last-applied-secret": "5e44c645bc123d23c86ea79cc836cb854ea07cac5341bafade616e18ec276303"} 2025-01-22T14:38:15.080Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "object": "some-name-pxc"} 2025-01-22T14:38:15.187Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "object": "some-name-proxysql"} 2025-01-22T14:38:17.278Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "d5789b98-9c07-4f13-b2ad-23e78d03839d", "error": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: failed to execute command in pod: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\nERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:339) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:925\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1025\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} 2025-01-22T14:38:26.334Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "88d98dc9-aa3e-4264-af25-14048dcf898e", "object": "some-name-pxc"} 2025-01-22T14:38:26.374Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "88d98dc9-aa3e-4264-af25-14048dcf898e", "object": "some-name-haproxy"} 2025-01-22T14:38:26.454Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "88d98dc9-aa3e-4264-af25-14048dcf898e", "object": "some-name-haproxy"} 2025-01-22T14:38:26.479Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "88d98dc9-aa3e-4264-af25-14048dcf898e", "object": "some-name-haproxy"} 2025-01-22T14:38:26.534Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "88d98dc9-aa3e-4264-af25-14048dcf898e", "object": "some-name-haproxy-replicas"} 2025-01-22T14:39:11.645Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4790f323-c7dd-45d1-8b60-07fdab72a379", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.users-729 on 10.94.144.10:53: no such host"} 2025-01-22T14:39:57.874Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "f1d8f92d-c4ef-4bbf-bbf2-458f627b1b84", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-0: dial tcp 10.231.0.9:33062: connect: connection refused"} 2025-01-22T14:40:33.828Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "root"} 2025-01-22T14:40:33.864Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "root"} 2025-01-22T14:40:33.873Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "secret": "some-name-mysql-init", "user": "root"} 2025-01-22T14:40:33.882Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "root"} 2025-01-22T14:40:33.921Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "root"} 2025-01-22T14:40:33.931Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "operator"} 2025-01-22T14:40:33.957Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "operator"} 2025-01-22T14:40:33.965Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "secret": "some-name-mysql-init", "user": "operator"} 2025-01-22T14:40:33.971Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "operator"} 2025-01-22T14:40:33.995Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "operator"} 2025-01-22T14:40:34.002Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "monitor"} 2025-01-22T14:40:34.026Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "monitor"} 2025-01-22T14:40:34.034Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "secret": "some-name-mysql-init", "user": "monitor"} 2025-01-22T14:40:34.041Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "monitor"} 2025-01-22T14:40:34.145Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "xtrabackup"} 2025-01-22T14:40:34.162Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "xtrabackup"} 2025-01-22T14:40:34.171Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "secret": "some-name-mysql-init", "user": "xtrabackup"} 2025-01-22T14:40:34.178Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "xtrabackup"} 2025-01-22T14:40:34.200Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "xtrabackup"} 2025-01-22T14:40:34.213Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "replication"} 2025-01-22T14:40:34.230Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "replication"} 2025-01-22T14:40:34.237Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "secret": "some-name-mysql-init", "user": "replication"} 2025-01-22T14:40:34.243Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "replication"} 2025-01-22T14:40:34.259Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "user": "replication"} 2025-01-22T14:40:34.259Z INFO PXC pods will be restarted {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"} 2025-01-22T14:40:34.264Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "4e016486-50f9-46bd-8fad-ba710403bff3", "object": "some-name-pxc"} 2025-01-22T14:41:30.414Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "c9694fcf-9c00-4728-be62-98b39233f7fe", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp 10.231.1.66:33062: connect: connection refused"} 2025-01-22T14:42:11.845Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "5911782e-a265-4028-bb70-795ef7d10224", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-729 on 10.94.144.10:53: no such host"} 2025-01-22T14:42:53.008Z INFO Password updated but old one not discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "6661aa3a-58a9-428c-bea2-523de76b58e9", "user": "monitor"} 2025-01-22T14:42:53.645Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "6661aa3a-58a9-428c-bea2-523de76b58e9", "user": "monitor"} 2025-01-22T14:42:57.831Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "5ee2d2db-c01d-4720-a71c-60082cb45b63", "user": "monitor"} 2025-01-22T14:42:57.854Z INFO Password updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "5ee2d2db-c01d-4720-a71c-60082cb45b63", "user": "monitor"} 2025-01-22T14:42:57.862Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "5ee2d2db-c01d-4720-a71c-60082cb45b63", "secret": "some-name-mysql-init", "user": "monitor"} 2025-01-22T14:42:57.870Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "5ee2d2db-c01d-4720-a71c-60082cb45b63", "user": "monitor"} 2025-01-22T14:43:03.139Z INFO Password updated but old one not discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "3a211d38-263f-44c0-aa7e-0e501f2a7048", "user": "monitor"} 2025-01-22T14:43:08.631Z INFO Password updated but old one not discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "bdf9eb06-6dc6-4a62-a75f-142a08581ea3", "user": "monitor"} 2025-01-22T14:43:13.934Z INFO Password updated but old one not discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "aa5d4ad4-563f-42a9-a2eb-7b713d7c4237", "user": "monitor"} 2025-01-22T14:43:19.498Z INFO Password updated but old one not discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "1827ffa2-f6aa-4b62-811a-b271fc29bf4f", "user": "monitor"} 2025-01-22T14:43:24.808Z INFO Password updated but old one not discarded {"controller": "pxc-controller", "namespace": "users-729", "name": "some-name", "reconcileID": "95a528ef-84ec-40a7-b556-a6d61eb12d4b", "user": "monitor"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1027 [mysql] 2025/01/22 14:32:27 connection.go:49: read tcp 10.231.1.58:41952->10.231.2.4:6032: read: connection reset by peer + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n users-729 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.zld1wsNLZT ++ mktemp + local LAST_ERR=/tmp/tmp.SdkrWXAhpg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zld1wsNLZT perconaxtradbcluster.pxc.percona.com "some-name" deleted + cat /tmp/tmp.SdkrWXAhpg + rm /tmp/tmp.zld1wsNLZT /tmp/tmp.SdkrWXAhpg + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.3wOBydD9vh ++ mktemp + local LAST_ERR=/tmp/tmp.oEYPELY97u + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3wOBydD9vh No resources found + cat /tmp/tmp.oEYPELY97u + rm /tmp/tmp.3wOBydD9vh /tmp/tmp.oEYPELY97u + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.AYeLI7ocSf ++ mktemp + local LAST_ERR=/tmp/tmp.2fqQN1Gb6Q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AYeLI7ocSf No resources found + cat /tmp/tmp.2fqQN1Gb6Q + rm /tmp/tmp.AYeLI7ocSf /tmp/tmp.2fqQN1Gb6Q + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.FB2wziZ7Zq ++ mktemp + local LAST_ERR=/tmp/tmp.ClVcUdoq4y + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FB2wziZ7Zq validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.ClVcUdoq4y + rm /tmp/tmp.FB2wziZ7Zq /tmp/tmp.ClVcUdoq4y + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.16.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace users-729 + rm -rf /tmp/tmp.HcH8ZcrDqq ++ mktemp + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.MYxxO64Ru0 + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- ++ mktemp + local LAST_OUT=/tmp/tmp.8psEIjkVyu + local LAST_ERR=/tmp/tmp.Hd7kEI5Yy3 + local exit_status=0 ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.K4ZY7hRXWe + for i in '$(seq 0 2)' + local exit_status=0 + set +e + kubectl delete --grace-period=0 --force=true namespace users-729 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator