Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/logs/users-5-7.log WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 + create_infra users-19584 + local ns=users-19584 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n users-4546 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.optYpmpGaI ++ mktemp + local LAST_ERR=/tmp/tmp.DkDKcWo7Un + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.optYpmpGaI perconaxtradbcluster.pxc.percona.com "some-name" deleted + cat /tmp/tmp.DkDKcWo7Un + rm /tmp/tmp.optYpmpGaI /tmp/tmp.DkDKcWo7Un + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.KdgDf7jtEX ++ mktemp + local LAST_ERR=/tmp/tmp.e4SxFmcMC2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KdgDf7jtEX No resources found + cat /tmp/tmp.e4SxFmcMC2 + rm /tmp/tmp.KdgDf7jtEX /tmp/tmp.e4SxFmcMC2 + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.izbg0qYFK3 ++ mktemp + local LAST_ERR=/tmp/tmp.RsxSCXWdMW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.izbg0qYFK3 No resources found + cat /tmp/tmp.RsxSCXWdMW + rm /tmp/tmp.izbg0qYFK3 /tmp/tmp.RsxSCXWdMW + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.bt9nPHfo88 ++ mktemp + local LAST_OUT=/tmp/tmp.TSh5XS4vS5 + xargs kubectl delete ns ++ mktemp + local LAST_ERR=/tmp/tmp.Ja6vpXzcfz + local exit_status=0 + local LAST_ERR=/tmp/tmp.76rUi2jUfZ + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bt9nPHfo88 + cat /tmp/tmp.Ja6vpXzcfz + rm /tmp/tmp.bt9nPHfo88 /tmp/tmp.Ja6vpXzcfz + return 0 namespace "users-4546" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TSh5XS4vS5 namespace "pxc-operator" deleted + cat /tmp/tmp.76rUi2jUfZ + rm /tmp/tmp.TSh5XS4vS5 /tmp/tmp.76rUi2jUfZ + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'namespace/pxc-operator - ' namespace/pxc-operator - + set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.moUuWUgWvn ++ mktemp + local LAST_ERR=/tmp/tmp.6MOtjRGTAa + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.moUuWUgWvn namespace/pxc-operator created + cat /tmp/tmp.6MOtjRGTAa + rm /tmp/tmp.moUuWUgWvn /tmp/tmp.6MOtjRGTAa + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.QjWBqARUHa +++ mktemp ++ local LAST_ERR=/tmp/tmp.gper9cePrx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QjWBqARUHa ++ cat /tmp/tmp.gper9cePrx ++ rm /tmp/tmp.QjWBqARUHa /tmp/tmp.gper9cePrx ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1747-942809ab-4-cluster9 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.KwN7ecljOI ++ mktemp + local LAST_ERR=/tmp/tmp.bIfHUgIB8Q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1747-942809ab-4-cluster9 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KwN7ecljOI Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1747-942809ab-4-cluster9" modified. + cat /tmp/tmp.bIfHUgIB8Q + rm /tmp/tmp.KwN7ecljOI /tmp/tmp.bIfHUgIB8Q + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.p4sywcxHy8 ++ mktemp + local LAST_ERR=/tmp/tmp.RLUwb9xMhP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.p4sywcxHy8 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.RLUwb9xMhP + rm /tmp/tmp.p4sywcxHy8 /tmp/tmp.RLUwb9xMhP + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.UIdEuDCsMU ++ mktemp + local LAST_ERR=/tmp/tmp.M0BRvu9dmy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UIdEuDCsMU clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.M0BRvu9dmy + rm /tmp/tmp.UIdEuDCsMU /tmp/tmp.M0BRvu9dmy + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1747-942809ab^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - ++ mktemp + local LAST_OUT=/tmp/tmp.IZd8RDYvU6 ++ mktemp + local LAST_ERR=/tmp/tmp.3b3qVBPhSd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IZd8RDYvU6 deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.3b3qVBPhSd + rm /tmp/tmp.IZd8RDYvU6 /tmp/tmp.3b3qVBPhSd + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.gY9xvHhGFY ++ mktemp + local LAST_ERR=/tmp/tmp.dn0oHx9hIk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gY9xvHhGFY pod/percona-xtradb-cluster-operator-598ffb648b-hkvvk condition met + cat /tmp/tmp.dn0oHx9hIk + rm /tmp/tmp.gY9xvHhGFY /tmp/tmp.dn0oHx9hIk + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.6ndr46kMVP +++ mktemp ++ local LAST_ERR=/tmp/tmp.Yw66b5IQ7M ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6ndr46kMVP ++ cat /tmp/tmp.Yw66b5IQ7M ++ rm /tmp/tmp.6ndr46kMVP /tmp/tmp.Yw66b5IQ7M ++ return 0 + wait_pod percona-xtradb-cluster-operator-598ffb648b-hkvvk 480 pxc-operator + local pod=percona-xtradb-cluster-operator-598ffb648b-hkvvk + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-598ffb648b-hkvvk ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-598ffb648b-hkvvk condition met percona-xtradb-cluster-operator-598ffb648b-hkvvk.Ok + sleep 3 + create_namespace users-19584 + local namespace=users-19584 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces users-19584' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces users-19584 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace users-19584 + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.5pcbkoIwXz ++ mktemp + local LAST_OUT=/tmp/tmp.g1ErKaH9Qq ++ mktemp + local LAST_ERR=/tmp/tmp.7fn7YX2J28 + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.vD28ghzmmj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace users-19584 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.g1ErKaH9Qq + cat /tmp/tmp.vD28ghzmmj + rm /tmp/tmp.g1ErKaH9Qq /tmp/tmp.vD28ghzmmj + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace users-19584 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace users-19584 Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.5pcbkoIwXz + cat /tmp/tmp.7fn7YX2J28 Error from server (NotFound): namespaces "users-19584" not found + rm /tmp/tmp.5pcbkoIwXz /tmp/tmp.7fn7YX2J28 + return 1 + : + wait_for_delete namespace/users-19584 + local res=namespace/users-19584 + echo -n 'namespace/users-19584 - ' namespace/users-19584 - + set +o xtrace Error from server (NotFound): namespaces "users-19584" not found + desc 'create namespace users-19584' + set +o xtrace ----------------------------------------------------------------------------------- create namespace users-19584 ----------------------------------------------------------------------------------- + kubectl_bin create namespace users-19584 ++ mktemp + local LAST_OUT=/tmp/tmp.tp9ei1UWHd ++ mktemp + local LAST_ERR=/tmp/tmp.TxpZ44DXic + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace users-19584 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tp9ei1UWHd namespace/users-19584 created + cat /tmp/tmp.TxpZ44DXic + rm /tmp/tmp.tp9ei1UWHd /tmp/tmp.TxpZ44DXic + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.4wtbc6jpmX +++ mktemp ++ local LAST_ERR=/tmp/tmp.irxw7QTQ6X ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4wtbc6jpmX ++ cat /tmp/tmp.irxw7QTQ6X ++ rm /tmp/tmp.4wtbc6jpmX /tmp/tmp.irxw7QTQ6X ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1747-942809ab-4-cluster9 --namespace=users-19584 ++ mktemp + local LAST_OUT=/tmp/tmp.On7Ry89yM6 ++ mktemp + local LAST_ERR=/tmp/tmp.v7xPMft8kg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1747-942809ab-4-cluster9 --namespace=users-19584 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.On7Ry89yM6 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1747-942809ab-4-cluster9" modified. + cat /tmp/tmp.v7xPMft8kg + rm /tmp/tmp.On7Ry89yM6 /tmp/tmp.v7xPMft8kg + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Up9JoaGhmD ++ mktemp + local LAST_ERR=/tmp/tmp.faLgpfkKXZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Up9JoaGhmD secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.faLgpfkKXZ + rm /tmp/tmp.Up9JoaGhmD /tmp/tmp.faLgpfkKXZ + return 0 + desc 'create PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC cluster ----------------------------------------------------------------------------------- + newpass=test-password ++ echo -n test-password ++ base64 + newpassencrypted=dGVzdC1wYXNzd29yZA== + cluster=some-name + spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/some-name.yml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/some-name.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.yxjbMYnsei ++ mktemp + local LAST_ERR=/tmp/tmp.uZGokhxtxs + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yxjbMYnsei secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.uZGokhxtxs + rm /tmp/tmp.yxjbMYnsei /tmp/tmp.uZGokhxtxs + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.FhAdZVJdBz + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-19584~ + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1747-942809ab#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.EaX1UrkUKQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FhAdZVJdBz deployment.apps/pxc-client created + cat /tmp/tmp.EaX1UrkUKQ + rm /tmp/tmp.FhAdZVJdBz /tmp/tmp.EaX1UrkUKQ + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + [[ some-name == \d\e\m\a\n\d\-\b\a\c\k\u\p ]] + [[ some-name == \d\e\m\a\n\d\-\b\a\c\k\u\p\-\c\l\o\u\d ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/some-name.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/some-name.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/conf/some-name.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-19584~ + local LAST_OUT=/tmp/tmp.CtGcC2GdfJ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.vItRytSXr8 + local exit_status=0 ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1747-942809ab#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CtGcC2GdfJ perconaxtradbcluster.pxc.percona.com/some-name created + cat /tmp/tmp.vItRytSXr8 + rm /tmp/tmp.CtGcC2GdfJ /tmp/tmp.vItRytSXr8 + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TSD1jTXPtf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wJID8bEZzi +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.TSD1jTXPtf +++ cat /tmp/tmp.wJID8bEZzi +++ rm /tmp/tmp.TSD1jTXPtf /tmp/tmp.wJID8bEZzi +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lA9lcg4mM1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JFdRP0D7KJ +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.lA9lcg4mM1 +++ cat /tmp/tmp.JFdRP0D7KJ +++ rm /tmp/tmp.lA9lcg4mM1 /tmp/tmp.JFdRP0D7KJ +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo some-name-proxysql ++ return + local proxy=some-name-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-19584 ++ mktemp + local LAST_OUT=/tmp/tmp.XtrRBUi1nJ ++ mktemp + local LAST_ERR=/tmp/tmp.oiexG4CC19 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-19584 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-19584 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n users-19584 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.XtrRBUi1nJ + cat /tmp/tmp.oiexG4CC19 error: no matching resources found + rm /tmp/tmp.XtrRBUi1nJ /tmp/tmp.oiexG4CC19 + return 1 + true + wait_for_running some-name-proxysql 1 + local name=some-name-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-proxysql-0 480 + local pod=some-name-proxysql-0 + local max_retry=480 + local ns= ++ echo some-name-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/some-name-proxysql-0 condition met some-name-proxysql-0.Ok + wait_for_running some-name-pxc 3 + local name=some-name-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ echo some-name-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met some-name-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-1 480 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met some-name-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-2 480 + local pod=some-name-pxc-2 + local max_retry=480 + local ns= ++ echo some-name-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-2 condition met some-name-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] ++ is_keyring_plugin_in_use some-name ++ local cluster=some-name ++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GQYslNHZ8I +++ mktemp ++ local LAST_ERR=/tmp/tmp.eTX1z0XZrB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GQYslNHZ8I ++ cat /tmp/tmp.eTX1z0XZrB Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.GQYslNHZ8I /tmp/tmp.eTX1z0XZrB ++ return 0 + [[ -n '' ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-proxysql -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h some-name-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZbbnHX1gHP +++ mktemp ++ local LAST_ERR=/tmp/tmp.flLQDfkLUs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZbbnHX1gHP ++ cat /tmp/tmp.flLQDfkLUs ++ rm /tmp/tmp.ZbbnHX1gHP /tmp/tmp.flLQDfkLUs ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-proxysql -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h some-name-proxysql -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.D0NzssuoDm +++ mktemp ++ local LAST_ERR=/tmp/tmp.6dFCf0bKju ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.D0NzssuoDm ++ cat /tmp/tmp.6dFCf0bKju ++ rm /tmp/tmp.D0NzssuoDm /tmp/tmp.6dFCf0bKju ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ehB3M7EL25 +++ mktemp ++ local LAST_ERR=/tmp/tmp.MEJaIjDwDF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ehB3M7EL25 ++ cat /tmp/tmp.MEJaIjDwDF ++ rm /tmp/tmp.ehB3M7EL25 /tmp/tmp.MEJaIjDwDF ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-1.sql /tmp/tmp.q8i5quND1N/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.frh9C34wxq +++ mktemp ++ local LAST_ERR=/tmp/tmp.i68hPTgJBj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.frh9C34wxq ++ cat /tmp/tmp.i68hPTgJBj ++ rm /tmp/tmp.frh9C34wxq /tmp/tmp.i68hPTgJBj ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-1.sql /tmp/tmp.q8i5quND1N/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KxTtSXdlOT +++ mktemp ++ local LAST_ERR=/tmp/tmp.PQLIgpkc08 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KxTtSXdlOT ++ cat /tmp/tmp.PQLIgpkc08 ++ rm /tmp/tmp.KxTtSXdlOT /tmp/tmp.PQLIgpkc08 ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-1.sql /tmp/tmp.q8i5quND1N/select-1.sql ++ is_keyring_plugin_in_use some-name ++ local cluster=some-name ++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PHiFfTGd7y +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ur1S6IhUB1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PHiFfTGd7y ++ cat /tmp/tmp.Ur1S6IhUB1 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.PHiFfTGd7y /tmp/tmp.Ur1S6IhUB1 ++ return 0 + '[' '' ']' + desc 'test root' + set +o xtrace ----------------------------------------------------------------------------------- test root ----------------------------------------------------------------------------------- + patch_secret my-cluster-secrets root dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=root + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.8VC6nvIq8u ++ mktemp + local LAST_ERR=/tmp/tmp.9YaCOHmaxZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"root": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8VC6nvIq8u secret/my-cluster-secrets patched + cat /tmp/tmp.9YaCOHmaxZ + rm /tmp/tmp.8VC6nvIq8u /tmp/tmp.9YaCOHmaxZ + return 0 + sleep 15 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uroot -p'\''test-password'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uroot -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xpPeUY9UdK +++ mktemp ++ local LAST_ERR=/tmp/tmp.thQHx4Vbna ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xpPeUY9UdK ++ cat /tmp/tmp.thQHx4Vbna ++ rm /tmp/tmp.xpPeUY9UdK /tmp/tmp.thQHx4Vbna ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-4.sql /tmp/tmp.q8i5quND1N/select-4.sql + desc 'test proxyadmin' + set +o xtrace ----------------------------------------------------------------------------------- test proxyadmin ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.9HpQHEe1wk ++ mktemp + local LAST_ERR=/tmp/tmp.U1HoCrPtEz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":3}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9HpQHEe1wk perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.U1HoCrPtEz + rm /tmp/tmp.9HpQHEe1wk /tmp/tmp.U1HoCrPtEz + return 0 + sleep 15 + wait_cluster_consistency some-name 3 3 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=3 + '[' -z 3 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EEjjoIsnsw +++ mktemp ++ local LAST_ERR=/tmp/tmp.XytTrJmjrP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EEjjoIsnsw ++ cat /tmp/tmp.XytTrJmjrP ++ rm /tmp/tmp.EEjjoIsnsw /tmp/tmp.XytTrJmjrP ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SoNiDHLhVb +++ mktemp ++ local LAST_ERR=/tmp/tmp.tJ0lonY7vQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SoNiDHLhVb ++ cat /tmp/tmp.tJ0lonY7vQ ++ rm /tmp/tmp.SoNiDHLhVb /tmp/tmp.tJ0lonY7vQ ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.27Zne62DVA ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.JkDPLtfbcX +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.27Zne62DVA +++++ cat /tmp/tmp.JkDPLtfbcX +++++ rm /tmp/tmp.27Zne62DVA /tmp/tmp.JkDPLtfbcX +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.e2AwQteLJC ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.AzbhZrTQkN +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.e2AwQteLJC +++++ cat /tmp/tmp.AzbhZrTQkN +++++ rm /tmp/tmp.e2AwQteLJC /tmp/tmp.AzbhZrTQkN +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GCy60pdsll +++ mktemp ++ local LAST_ERR=/tmp/tmp.ENMIWZCQpf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GCy60pdsll ++ cat /tmp/tmp.ENMIWZCQpf ++ rm /tmp/tmp.GCy60pdsll /tmp/tmp.ENMIWZCQpf ++ return 0 + [[ 3 == \3 ]] + patch_secret my-cluster-secrets proxyadmin dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=proxyadmin + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.hmuLWOZpPJ ++ mktemp + local LAST_ERR=/tmp/tmp.TtNIkyhdFs + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"proxyadmin": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hmuLWOZpPJ secret/my-cluster-secrets patched + cat /tmp/tmp.TtNIkyhdFs + rm /tmp/tmp.hmuLWOZpPJ /tmp/tmp.TtNIkyhdFs + return 0 + sleep 15 + wait_cluster_consistency some-name 3 3 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=3 + '[' -z 3 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gXJavvadxM +++ mktemp ++ local LAST_ERR=/tmp/tmp.831aCAWJSu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gXJavvadxM ++ cat /tmp/tmp.831aCAWJSu ++ rm /tmp/tmp.gXJavvadxM /tmp/tmp.831aCAWJSu ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.G72ZJkWrlw +++ mktemp ++ local LAST_ERR=/tmp/tmp.mz8nlVgRYA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.G72ZJkWrlw ++ cat /tmp/tmp.mz8nlVgRYA ++ rm /tmp/tmp.G72ZJkWrlw /tmp/tmp.mz8nlVgRYA ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IonNVXmtqW +++ mktemp ++ local LAST_ERR=/tmp/tmp.5CCOScZnNy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IonNVXmtqW ++ cat /tmp/tmp.5CCOScZnNy ++ rm /tmp/tmp.IonNVXmtqW /tmp/tmp.5CCOScZnNy ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZnPOK248YJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.yCbWDUNnDR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZnPOK248YJ ++ cat /tmp/tmp.yCbWDUNnDR ++ rm /tmp/tmp.ZnPOK248YJ /tmp/tmp.yCbWDUNnDR ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.cECVoOpx04 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.YexZgxWAYb +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.cECVoOpx04 +++++ cat /tmp/tmp.YexZgxWAYb +++++ rm /tmp/tmp.cECVoOpx04 /tmp/tmp.YexZgxWAYb +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.WFIHJpNjH9 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.RAZtMqTaIO +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.WFIHJpNjH9 +++++ cat /tmp/tmp.RAZtMqTaIO +++++ rm /tmp/tmp.WFIHJpNjH9 /tmp/tmp.RAZtMqTaIO +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XaoYs8Zp1y +++ mktemp ++ local LAST_ERR=/tmp/tmp.dKJkbKLWRJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XaoYs8Zp1y ++ cat /tmp/tmp.dKJkbKLWRJ ++ rm /tmp/tmp.XaoYs8Zp1y /tmp/tmp.dKJkbKLWRJ ++ return 0 + [[ 3 == \3 ]] + compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 '' proxysql + local command_id=select-2 + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-0 + local postfix= + local container_name=proxysql + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-0 proxysql + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-0 + local container_name=proxysql + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-2.sql /tmp/tmp.q8i5quND1N/select-2.sql + compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 '' proxysql + local command_id=select-2 + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-1 + local postfix= + local container_name=proxysql + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-1 proxysql + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-1 + local container_name=proxysql + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-2.sql /tmp/tmp.q8i5quND1N/select-2.sql + compare_mysql_cmd_local select-2 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 '' proxysql + local command_id=select-2 + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-2 + local postfix= + local container_name=proxysql + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql_local 'SHOW TABLES;' '-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' some-name-proxysql-2 proxysql + local 'command=SHOW TABLES;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -p'\''test-password'\''' + local pod=some-name-proxysql-2 + local container_name=proxysql + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-2.sql /tmp/tmp.q8i5quND1N/select-2.sql + desc 'test xtrabackup' + set +o xtrace ----------------------------------------------------------------------------------- test xtrabackup ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.ImHxPUEV9V ++ mktemp + local LAST_ERR=/tmp/tmp.okyuMvWC1z + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type=merge '-p={"spec":{"proxysql":{"size":2}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ImHxPUEV9V perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.okyuMvWC1z + rm /tmp/tmp.ImHxPUEV9V /tmp/tmp.okyuMvWC1z + return 0 + patch_secret my-cluster-secrets xtrabackup dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=xtrabackup + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.L89BaPyu8D ++ mktemp + local LAST_ERR=/tmp/tmp.TwgsPvLgD1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"xtrabackup": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.L89BaPyu8D secret/my-cluster-secrets patched + cat /tmp/tmp.TwgsPvLgD1 + rm /tmp/tmp.L89BaPyu8D /tmp/tmp.TwgsPvLgD1 + return 0 + sleep 15 + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tP6kNIKGOs +++ mktemp ++ local LAST_ERR=/tmp/tmp.Xpy3M78k4F ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tP6kNIKGOs ++ cat /tmp/tmp.Xpy3M78k4F ++ rm /tmp/tmp.tP6kNIKGOs /tmp/tmp.Xpy3M78k4F ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JXtMMQa1UG +++ mktemp ++ local LAST_ERR=/tmp/tmp.8I3p8L3aqm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JXtMMQa1UG ++ cat /tmp/tmp.8I3p8L3aqm ++ rm /tmp/tmp.JXtMMQa1UG /tmp/tmp.8I3p8L3aqm ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.D7HyICMYgp +++ mktemp ++ local LAST_ERR=/tmp/tmp.WuXudvSl8k ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.D7HyICMYgp ++ cat /tmp/tmp.WuXudvSl8k ++ rm /tmp/tmp.D7HyICMYgp /tmp/tmp.WuXudvSl8k ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.77sP07x6oW +++ mktemp ++ local LAST_ERR=/tmp/tmp.mJ9S1W3ROw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.77sP07x6oW ++ cat /tmp/tmp.mJ9S1W3ROw ++ rm /tmp/tmp.77sP07x6oW /tmp/tmp.mJ9S1W3ROw ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7Vm45y95IR +++ mktemp ++ local LAST_ERR=/tmp/tmp.8JRAie3T5F ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7Vm45y95IR ++ cat /tmp/tmp.8JRAie3T5F ++ rm /tmp/tmp.7Vm45y95IR /tmp/tmp.8JRAie3T5F ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YtQoNT3i0r +++ mktemp ++ local LAST_ERR=/tmp/tmp.SaH5EjNcjG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YtQoNT3i0r ++ cat /tmp/tmp.SaH5EjNcjG ++ rm /tmp/tmp.YtQoNT3i0r /tmp/tmp.SaH5EjNcjG ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QktstgS4fN +++ mktemp ++ local LAST_ERR=/tmp/tmp.urtNhD77DQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QktstgS4fN ++ cat /tmp/tmp.urtNhD77DQ ++ rm /tmp/tmp.QktstgS4fN /tmp/tmp.urtNhD77DQ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cqMJUZ06As +++ mktemp ++ local LAST_ERR=/tmp/tmp.PGEd7cWPPH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cqMJUZ06As ++ cat /tmp/tmp.PGEd7cWPPH ++ rm /tmp/tmp.cqMJUZ06As /tmp/tmp.PGEd7cWPPH ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Nqs7QlLgBe +++ mktemp ++ local LAST_ERR=/tmp/tmp.emdiJ1JOnT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Nqs7QlLgBe ++ cat /tmp/tmp.emdiJ1JOnT ++ rm /tmp/tmp.Nqs7QlLgBe /tmp/tmp.emdiJ1JOnT ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hc8L16A1BF +++ mktemp ++ local LAST_ERR=/tmp/tmp.HHI1yyZbGP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hc8L16A1BF ++ cat /tmp/tmp.HHI1yyZbGP ++ rm /tmp/tmp.hc8L16A1BF /tmp/tmp.HHI1yyZbGP ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.u0pwNTeoDR ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.wUl7hNf39W +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.u0pwNTeoDR +++++ cat /tmp/tmp.wUl7hNf39W +++++ rm /tmp/tmp.u0pwNTeoDR /tmp/tmp.wUl7hNf39W +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.tOVS6DKZ4G ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.RWxQ55J8Of +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.tOVS6DKZ4G +++++ cat /tmp/tmp.RWxQ55J8Of +++++ rm /tmp/tmp.tOVS6DKZ4G /tmp/tmp.RWxQ55J8Of +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bCjXOehHr3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mmpdHSLJGp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bCjXOehHr3 ++ cat /tmp/tmp.mmpdHSLJGp ++ rm /tmp/tmp.bCjXOehHr3 /tmp/tmp.mmpdHSLJGp ++ return 0 + [[ 2 == \2 ]] + compare_mysql_cmd_local select-3 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 '' pxc + local command_id=select-3 + local 'command=SHOW DATABASES;' + local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' + local pod=some-name-pxc-0 + local postfix= + local container_name=pxc + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql_local 'SHOW DATABASES;' '-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' some-name-pxc-0 pxc + local 'command=SHOW DATABASES;' + local 'uri=-h 127.0.0.1 -uxtrabackup -p'\''test-password'\''' + local pod=some-name-pxc-0 + local container_name=pxc + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-3.sql /tmp/tmp.q8i5quND1N/select-3.sql + desc 'test monitor' + set +o xtrace ----------------------------------------------------------------------------------- test monitor ----------------------------------------------------------------------------------- + patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=monitor + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.AWRLFN9EW0 ++ mktemp + local LAST_ERR=/tmp/tmp.bil7pHo5wf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AWRLFN9EW0 secret/my-cluster-secrets patched + cat /tmp/tmp.bil7pHo5wf + rm /tmp/tmp.AWRLFN9EW0 /tmp/tmp.bil7pHo5wf + return 0 + wait_for_password_propagation my-cluster-secrets monitor + local secret=my-cluster-secrets + local user=monitor + local max_retry=240 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.dDEgyV56Ru +++ mktemp ++ local LAST_ERR=/tmp/tmp.t9psSs9ZN6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dDEgyV56Ru ++ cat /tmp/tmp.t9psSs9ZN6 ++ rm /tmp/tmp.dDEgyV56Ru /tmp/tmp.t9psSs9ZN6 ++ return 0 + local root_pass=test-password + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 5\.7 ]] + echo 'Skipping dual password feature doesn'\''t work for 5.7. PXC 5.7 doesn'\''t support it!' Skipping dual password feature doesn't work for 5.7. PXC 5.7 doesn't support it! + return + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NbnDAsJ13c +++ mktemp ++ local LAST_ERR=/tmp/tmp.wKJdNi76Nh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NbnDAsJ13c ++ cat /tmp/tmp.wKJdNi76Nh ++ rm /tmp/tmp.NbnDAsJ13c /tmp/tmp.wKJdNi76Nh ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QualJgkpKf +++ mktemp ++ local LAST_ERR=/tmp/tmp.SaBmRIglGl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QualJgkpKf ++ cat /tmp/tmp.SaBmRIglGl ++ rm /tmp/tmp.QualJgkpKf /tmp/tmp.SaBmRIglGl ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AywSent0Ec +++ mktemp ++ local LAST_ERR=/tmp/tmp.tQJGcC7x6t ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AywSent0Ec ++ cat /tmp/tmp.tQJGcC7x6t ++ rm /tmp/tmp.AywSent0Ec /tmp/tmp.tQJGcC7x6t ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tBh3bmrSMI +++ mktemp ++ local LAST_ERR=/tmp/tmp.cO1sNxllCM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tBh3bmrSMI ++ cat /tmp/tmp.cO1sNxllCM ++ rm /tmp/tmp.tBh3bmrSMI /tmp/tmp.cO1sNxllCM ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.pjEUwJO9ZO ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.LTbd8Xk0ex +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.pjEUwJO9ZO +++++ cat /tmp/tmp.LTbd8Xk0ex +++++ rm /tmp/tmp.pjEUwJO9ZO /tmp/tmp.LTbd8Xk0ex +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.am3vz8sZRq ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Rjv4JWQvyC +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.am3vz8sZRq +++++ cat /tmp/tmp.Rjv4JWQvyC +++++ rm /tmp/tmp.am3vz8sZRq /tmp/tmp.Rjv4JWQvyC +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DLk5KbKNPg +++ mktemp ++ local LAST_ERR=/tmp/tmp.yyHXEjKuT5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DLk5KbKNPg ++ cat /tmp/tmp.yyHXEjKuT5 ++ rm /tmp/tmp.DLk5KbKNPg /tmp/tmp.yyHXEjKuT5 ++ return 0 + [[ 2 == \2 ]] + sleep 10 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -umonitor -p'\''test-password'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -umonitor -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gAZgfO0MmL +++ mktemp ++ local LAST_ERR=/tmp/tmp.m9OzUQp9Yd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gAZgfO0MmL ++ cat /tmp/tmp.m9OzUQp9Yd ++ rm /tmp/tmp.gAZgfO0MmL /tmp/tmp.m9OzUQp9Yd ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-4.sql /tmp/tmp.q8i5quND1N/select-4.sql + desc 'test operator' + set +o xtrace ----------------------------------------------------------------------------------- test operator ----------------------------------------------------------------------------------- + patch_secret my-cluster-secrets operator dGVzdC1wYXNzd29yZA== + local secret=my-cluster-secrets + local key=operator + local value=dGVzdC1wYXNzd29yZA== + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.jn8TDCffAO ++ mktemp + local LAST_ERR=/tmp/tmp.yZJqcRSXL4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"operator": "dGVzdC1wYXNzd29yZA=="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jn8TDCffAO secret/my-cluster-secrets patched + cat /tmp/tmp.yZJqcRSXL4 + rm /tmp/tmp.jn8TDCffAO /tmp/tmp.yZJqcRSXL4 + return 0 + sleep 15 + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ALTWEniogX +++ mktemp ++ local LAST_ERR=/tmp/tmp.1B8t76aNfs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ALTWEniogX ++ cat /tmp/tmp.1B8t76aNfs ++ rm /tmp/tmp.ALTWEniogX /tmp/tmp.1B8t76aNfs ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v7ocZlPkoo +++ mktemp ++ local LAST_ERR=/tmp/tmp.TJs5f2yYRZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.v7ocZlPkoo ++ cat /tmp/tmp.TJs5f2yYRZ ++ rm /tmp/tmp.v7ocZlPkoo /tmp/tmp.TJs5f2yYRZ ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.18CaDDAXt8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ntiKYBh6P9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.18CaDDAXt8 ++ cat /tmp/tmp.ntiKYBh6P9 ++ rm /tmp/tmp.18CaDDAXt8 /tmp/tmp.ntiKYBh6P9 ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.kRwEf2qHcY ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ESpNGY93I7 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.kRwEf2qHcY +++++ cat /tmp/tmp.ESpNGY93I7 +++++ rm /tmp/tmp.kRwEf2qHcY /tmp/tmp.ESpNGY93I7 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.mIw9fIdpha ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.EcNULsgzOi +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.mIw9fIdpha +++++ cat /tmp/tmp.EcNULsgzOi +++++ rm /tmp/tmp.mIw9fIdpha /tmp/tmp.EcNULsgzOi +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KqFcXVjtuC +++ mktemp ++ local LAST_ERR=/tmp/tmp.ocsXO12Znm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KqFcXVjtuC ++ cat /tmp/tmp.ocsXO12Znm ++ rm /tmp/tmp.KqFcXVjtuC /tmp/tmp.ocsXO12Znm ++ return 0 + [[ 2 == \2 ]] + sleep 10 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pKgcf6cw3a +++ mktemp ++ local LAST_ERR=/tmp/tmp.jRsPU1tImk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pKgcf6cw3a ++ cat /tmp/tmp.jRsPU1tImk ++ rm /tmp/tmp.pKgcf6cw3a /tmp/tmp.jRsPU1tImk ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-4.sql /tmp/tmp.q8i5quND1N/select-4.sql + desc 'change secret name' + set +o xtrace ----------------------------------------------------------------------------------- change secret name ----------------------------------------------------------------------------------- + kubectl_bin patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.BjkOPkX4DG ++ mktemp + local LAST_ERR=/tmp/tmp.YIrmPQ2KQH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc some-name --type merge --patch '{"spec": {"secretsName":"my-cluster-secrets-2"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BjkOPkX4DG perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.YIrmPQ2KQH + rm /tmp/tmp.BjkOPkX4DG /tmp/tmp.YIrmPQ2KQH + return 0 + sleep 30 + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZgKMbHjzYU +++ mktemp ++ local LAST_ERR=/tmp/tmp.mkUF9PYF1D ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZgKMbHjzYU ++ cat /tmp/tmp.mkUF9PYF1D ++ rm /tmp/tmp.ZgKMbHjzYU /tmp/tmp.mkUF9PYF1D ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lj7OoFlnhC +++ mktemp ++ local LAST_ERR=/tmp/tmp.o101HQYRfH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lj7OoFlnhC ++ cat /tmp/tmp.o101HQYRfH ++ rm /tmp/tmp.lj7OoFlnhC /tmp/tmp.o101HQYRfH ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NiwiIs73Xo +++ mktemp ++ local LAST_ERR=/tmp/tmp.qNvuZylj9v ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NiwiIs73Xo ++ cat /tmp/tmp.qNvuZylj9v ++ rm /tmp/tmp.NiwiIs73Xo /tmp/tmp.qNvuZylj9v ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HwVSYH4jWX +++ mktemp ++ local LAST_ERR=/tmp/tmp.V21BeG9RB3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HwVSYH4jWX ++ cat /tmp/tmp.V21BeG9RB3 ++ rm /tmp/tmp.HwVSYH4jWX /tmp/tmp.V21BeG9RB3 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.STi2p1s2xN +++ mktemp ++ local LAST_ERR=/tmp/tmp.4woMqGqJvi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.STi2p1s2xN ++ cat /tmp/tmp.4woMqGqJvi ++ rm /tmp/tmp.STi2p1s2xN /tmp/tmp.4woMqGqJvi ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LVS5n0ywOy +++ mktemp ++ local LAST_ERR=/tmp/tmp.AtnNTG0SnP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LVS5n0ywOy ++ cat /tmp/tmp.AtnNTG0SnP ++ rm /tmp/tmp.LVS5n0ywOy /tmp/tmp.AtnNTG0SnP ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1L2bfZcJ6y +++ mktemp ++ local LAST_ERR=/tmp/tmp.ihHNjUKmCT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1L2bfZcJ6y ++ cat /tmp/tmp.ihHNjUKmCT ++ rm /tmp/tmp.1L2bfZcJ6y /tmp/tmp.ihHNjUKmCT ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bXkjgpE0mp +++ mktemp ++ local LAST_ERR=/tmp/tmp.4CYs0Iwlp8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bXkjgpE0mp ++ cat /tmp/tmp.4CYs0Iwlp8 ++ rm /tmp/tmp.bXkjgpE0mp /tmp/tmp.4CYs0Iwlp8 ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.mJmWrSMt0I ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.M5b7LGArS3 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.mJmWrSMt0I +++++ cat /tmp/tmp.M5b7LGArS3 +++++ rm /tmp/tmp.mJmWrSMt0I /tmp/tmp.M5b7LGArS3 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.gUlkitVww0 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.4toHyRHytD +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.gUlkitVww0 +++++ cat /tmp/tmp.4toHyRHytD +++++ rm /tmp/tmp.gUlkitVww0 /tmp/tmp.4toHyRHytD +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RNTsvrRjic +++ mktemp ++ local LAST_ERR=/tmp/tmp.9FfKyaaHFU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RNTsvrRjic ++ cat /tmp/tmp.9FfKyaaHFU ++ rm /tmp/tmp.RNTsvrRjic /tmp/tmp.9FfKyaaHFU ++ return 0 + [[ 2 == \2 ]] + desc 'test new operator' + set +o xtrace ----------------------------------------------------------------------------------- test new operator ----------------------------------------------------------------------------------- + newpass=test-password2 ++ echo -n test-password2 ++ base64 + newpassencrypted=dGVzdC1wYXNzd29yZDI= + patch_secret my-cluster-secrets-2 operator dGVzdC1wYXNzd29yZDI= + local secret=my-cluster-secrets-2 + local key=operator + local value=dGVzdC1wYXNzd29yZDI= + kubectl_bin patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.3lbX8JGDW6 ++ mktemp + local LAST_ERR=/tmp/tmp.3d7R7rto5X + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets-2 '-p={"data":{"operator": "dGVzdC1wYXNzd29yZDI="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3lbX8JGDW6 secret/my-cluster-secrets-2 patched + cat /tmp/tmp.3d7R7rto5X + rm /tmp/tmp.3lbX8JGDW6 /tmp/tmp.3d7R7rto5X + return 0 + sleep 15 + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vtUTPIVnfk +++ mktemp ++ local LAST_ERR=/tmp/tmp.colC5D5b0w ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vtUTPIVnfk ++ cat /tmp/tmp.colC5D5b0w ++ rm /tmp/tmp.vtUTPIVnfk /tmp/tmp.colC5D5b0w ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XHolkgfq4o +++ mktemp ++ local LAST_ERR=/tmp/tmp.0edJFjicBq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XHolkgfq4o ++ cat /tmp/tmp.0edJFjicBq ++ rm /tmp/tmp.XHolkgfq4o /tmp/tmp.0edJFjicBq ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U5TLCNi2ic +++ mktemp ++ local LAST_ERR=/tmp/tmp.2U27igKRie ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.U5TLCNi2ic ++ cat /tmp/tmp.2U27igKRie ++ rm /tmp/tmp.U5TLCNi2ic /tmp/tmp.2U27igKRie ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.8oRJXsfmsF ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.5nKkFw8Dwe +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.8oRJXsfmsF +++++ cat /tmp/tmp.5nKkFw8Dwe +++++ rm /tmp/tmp.8oRJXsfmsF /tmp/tmp.5nKkFw8Dwe +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.9lTttb50KJ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.JYV9qlPu3H +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.9lTttb50KJ +++++ cat /tmp/tmp.JYV9qlPu3H +++++ rm /tmp/tmp.9lTttb50KJ /tmp/tmp.JYV9qlPu3H +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LnV375EgYa +++ mktemp ++ local LAST_ERR=/tmp/tmp.gRPOP1BnPe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LnV375EgYa ++ cat /tmp/tmp.gRPOP1BnPe ++ rm /tmp/tmp.LnV375EgYa /tmp/tmp.gRPOP1BnPe ++ return 0 + [[ 2 == \2 ]] + sleep 20 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oVY2smeXD2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.AbefUrD9C9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oVY2smeXD2 ++ cat /tmp/tmp.AbefUrD9C9 ++ rm /tmp/tmp.oVY2smeXD2 /tmp/tmp.AbefUrD9C9 ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-4.sql /tmp/tmp.q8i5quND1N/select-4.sql ++ getSecretData my-cluster-secrets-2 root ++ local secretName=my-cluster-secrets-2 ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets-2 '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.9dcRRGe9zl +++ mktemp ++ local LAST_ERR=/tmp/tmp.iA6yLFzcqs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets-2 '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9dcRRGe9zl ++ cat /tmp/tmp.iA6yLFzcqs ++ rm /tmp/tmp.9dcRRGe9zl /tmp/tmp.iA6yLFzcqs ++ return 0 + newpass=7CEwsLdICTPh8-vF + desc 'test new users sync' + set +o xtrace ----------------------------------------------------------------------------------- test new users sync ----------------------------------------------------------------------------------- + run_mysql 'CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\''7CEwsLdICTPh8-vF'\'';' '-h some-name-pxc -uroot -p'\''7CEwsLdICTPh8-vF'\''' + local 'command=CREATE USER '\''testsync'\''@'\''%'\'' IDENTIFIED BY '\''7CEwsLdICTPh8-vF'\'';' + local 'uri=-h some-name-pxc -uroot -p'\''7CEwsLdICTPh8-vF'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zbikEdqAsg +++ mktemp ++ local LAST_ERR=/tmp/tmp.fssd8fSCo4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zbikEdqAsg ++ cat /tmp/tmp.fssd8fSCo4 ++ rm /tmp/tmp.zbikEdqAsg /tmp/tmp.fssd8fSCo4 ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + sleep 40 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\''7CEwsLdICTPh8-vF'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -utestsync -p'\''7CEwsLdICTPh8-vF'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -utestsync -p'\''7CEwsLdICTPh8-vF'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -utestsync -p'\''7CEwsLdICTPh8-vF'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EDfIzaDwzR +++ mktemp ++ local LAST_ERR=/tmp/tmp.HmZ6SZiJ0n ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EDfIzaDwzR ++ cat /tmp/tmp.HmZ6SZiJ0n ++ rm /tmp/tmp.EDfIzaDwzR /tmp/tmp.HmZ6SZiJ0n ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-4.sql /tmp/tmp.q8i5quND1N/select-4.sql ++ getSecretData internal-some-name operator ++ local secretName=internal-some-name ++ local dataKey=operator ++ kubectl_bin get secrets/internal-some-name '--template={{.data.operator}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.x4q5PkPAuc +++ mktemp ++ local LAST_ERR=/tmp/tmp.AVIMisTfcg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/internal-some-name '--template={{.data.operator}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.x4q5PkPAuc ++ cat /tmp/tmp.AVIMisTfcg ++ rm /tmp/tmp.x4q5PkPAuc /tmp/tmp.AVIMisTfcg ++ return 0 + pass=test-password2 + desc 'check secret without operator' + set +o xtrace ----------------------------------------------------------------------------------- check secret without operator ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.8xYcYg8O4l ++ mktemp + local LAST_ERR=/tmp/tmp.bNQuTZ3P1V + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8xYcYg8O4l secret/my-cluster-secrets-2 configured + cat /tmp/tmp.bNQuTZ3P1V Warning: resource secrets/my-cluster-secrets-2 is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.8xYcYg8O4l /tmp/tmp.bNQuTZ3P1V + return 0 + sleep 15 + compare_mysql_cmd select-4 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local command_id=select-4 + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SHOW TABLES;' '-h some-name-proxysql -uoperator -p'\''test-password2'\''' + local 'command=SHOW TABLES;' + local 'uri=-h some-name-proxysql -uoperator -p'\''test-password2'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZgkwX8erbr +++ mktemp ++ local LAST_ERR=/tmp/tmp.2A3G2p519h ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZgkwX8erbr ++ cat /tmp/tmp.2A3G2p519h ++ rm /tmp/tmp.ZgkwX8erbr /tmp/tmp.2A3G2p519h ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-4.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-4.sql /tmp/tmp.q8i5quND1N/select-4.sql + newpass=test-password2 ++ echo -n test-password2 ++ base64 + newpassencrypted=dGVzdC1wYXNzd29yZDI= + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/conf/some-name.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1747-942809ab#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc5.7#' + local LAST_OUT=/tmp/tmp.njzvjt0sDd + /usr/bin/sed -e s~minio-service.#namespace~minio-service.users-19584~ + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + local LAST_ERR=/tmp/tmp.pjggc9CgFO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.njzvjt0sDd perconaxtradbcluster.pxc.percona.com/some-name configured + cat /tmp/tmp.pjggc9CgFO + rm /tmp/tmp.njzvjt0sDd /tmp/tmp.pjggc9CgFO + return 0 + sleep 15 + wait_cluster_consistency some-name 3 3 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=3 + '[' -z 3 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EDm0SqW34o +++ mktemp ++ local LAST_ERR=/tmp/tmp.JIwqUtByxS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EDm0SqW34o ++ cat /tmp/tmp.JIwqUtByxS ++ rm /tmp/tmp.EDm0SqW34o /tmp/tmp.JIwqUtByxS ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5lojq30OgJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.0oFg8l5EGR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5lojq30OgJ ++ cat /tmp/tmp.0oFg8l5EGR ++ rm /tmp/tmp.5lojq30OgJ /tmp/tmp.0oFg8l5EGR ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dX4cICivW1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.VbGf6DSZd7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dX4cICivW1 ++ cat /tmp/tmp.VbGf6DSZd7 ++ rm /tmp/tmp.dX4cICivW1 /tmp/tmp.VbGf6DSZd7 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iI4PJw8HR4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.pgVodo7ECS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iI4PJw8HR4 ++ cat /tmp/tmp.pgVodo7ECS ++ rm /tmp/tmp.iI4PJw8HR4 /tmp/tmp.pgVodo7ECS ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9UijDf6nzY +++ mktemp ++ local LAST_ERR=/tmp/tmp.z4qeO9xSu7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9UijDf6nzY ++ cat /tmp/tmp.z4qeO9xSu7 ++ rm /tmp/tmp.9UijDf6nzY /tmp/tmp.z4qeO9xSu7 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.a5O9BcAh4g +++ mktemp ++ local LAST_ERR=/tmp/tmp.Igu6Il5c6O ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.a5O9BcAh4g ++ cat /tmp/tmp.Igu6Il5c6O ++ rm /tmp/tmp.a5O9BcAh4g /tmp/tmp.Igu6Il5c6O ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lAlobi42RJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.T6xNPonFHf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lAlobi42RJ ++ cat /tmp/tmp.T6xNPonFHf ++ rm /tmp/tmp.lAlobi42RJ /tmp/tmp.T6xNPonFHf ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yGwJ0tug5K +++ mktemp ++ local LAST_ERR=/tmp/tmp.zkcyCcpt0F ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yGwJ0tug5K ++ cat /tmp/tmp.zkcyCcpt0F ++ rm /tmp/tmp.yGwJ0tug5K /tmp/tmp.zkcyCcpt0F ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HFLw7KGqXv +++ mktemp ++ local LAST_ERR=/tmp/tmp.HvutbnWE8c ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HFLw7KGqXv ++ cat /tmp/tmp.HvutbnWE8c ++ rm /tmp/tmp.HFLw7KGqXv /tmp/tmp.HvutbnWE8c ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 8 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OmLEgjoA7w +++ mktemp ++ local LAST_ERR=/tmp/tmp.EogiXtLdPm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OmLEgjoA7w ++ cat /tmp/tmp.EogiXtLdPm ++ rm /tmp/tmp.OmLEgjoA7w /tmp/tmp.EogiXtLdPm ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 9 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zHQqe9rOod +++ mktemp ++ local LAST_ERR=/tmp/tmp.SaJ0MqCZ7v ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zHQqe9rOod ++ cat /tmp/tmp.SaJ0MqCZ7v ++ rm /tmp/tmp.zHQqe9rOod /tmp/tmp.SaJ0MqCZ7v ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 10 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nrAaVTdtJj +++ mktemp ++ local LAST_ERR=/tmp/tmp.dQoXcO9koe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nrAaVTdtJj ++ cat /tmp/tmp.dQoXcO9koe ++ rm /tmp/tmp.nrAaVTdtJj /tmp/tmp.dQoXcO9koe ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 11 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1R3Dz6FDrm +++ mktemp ++ local LAST_ERR=/tmp/tmp.wwyxPLf4Wd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1R3Dz6FDrm ++ cat /tmp/tmp.wwyxPLf4Wd ++ rm /tmp/tmp.1R3Dz6FDrm /tmp/tmp.wwyxPLf4Wd ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 12 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4cHoIjEOMM +++ mktemp ++ local LAST_ERR=/tmp/tmp.adz6my992o ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4cHoIjEOMM ++ cat /tmp/tmp.adz6my992o ++ rm /tmp/tmp.4cHoIjEOMM /tmp/tmp.adz6my992o ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 13 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rwzguxvlex +++ mktemp ++ local LAST_ERR=/tmp/tmp.eMNkAd5EuP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rwzguxvlex ++ cat /tmp/tmp.eMNkAd5EuP ++ rm /tmp/tmp.rwzguxvlex /tmp/tmp.eMNkAd5EuP ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 14 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.86PZHWiplZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.i2pxD69NBj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.86PZHWiplZ ++ cat /tmp/tmp.i2pxD69NBj ++ rm /tmp/tmp.86PZHWiplZ /tmp/tmp.i2pxD69NBj ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 15 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MxRA0zyA5O +++ mktemp ++ local LAST_ERR=/tmp/tmp.ybZBfDr4Rq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MxRA0zyA5O ++ cat /tmp/tmp.ybZBfDr4Rq ++ rm /tmp/tmp.MxRA0zyA5O /tmp/tmp.ybZBfDr4Rq ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 16 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.btNxseVzXT +++ mktemp ++ local LAST_ERR=/tmp/tmp.2cxbQfaYgo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.btNxseVzXT ++ cat /tmp/tmp.2cxbQfaYgo ++ rm /tmp/tmp.btNxseVzXT /tmp/tmp.2cxbQfaYgo ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 17 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w0bLzOO7cT +++ mktemp ++ local LAST_ERR=/tmp/tmp.UMgACtvPWy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w0bLzOO7cT ++ cat /tmp/tmp.UMgACtvPWy ++ rm /tmp/tmp.w0bLzOO7cT /tmp/tmp.UMgACtvPWy ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 18 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iOQL0xw1lK +++ mktemp ++ local LAST_ERR=/tmp/tmp.pqww2JEWj4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iOQL0xw1lK ++ cat /tmp/tmp.pqww2JEWj4 ++ rm /tmp/tmp.iOQL0xw1lK /tmp/tmp.pqww2JEWj4 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 19 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.moGKZhACYk +++ mktemp ++ local LAST_ERR=/tmp/tmp.AAaOPHGM0C ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.moGKZhACYk ++ cat /tmp/tmp.AAaOPHGM0C ++ rm /tmp/tmp.moGKZhACYk /tmp/tmp.AAaOPHGM0C ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 20 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dXulPgBHDB +++ mktemp ++ local LAST_ERR=/tmp/tmp.IHuMH7Q0oM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dXulPgBHDB ++ cat /tmp/tmp.IHuMH7Q0oM ++ rm /tmp/tmp.dXulPgBHDB /tmp/tmp.IHuMH7Q0oM ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.B9w6MIZaE3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.E6WKLBAnoD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.B9w6MIZaE3 ++ cat /tmp/tmp.E6WKLBAnoD ++ rm /tmp/tmp.B9w6MIZaE3 /tmp/tmp.E6WKLBAnoD ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.dX1iA91hD9 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.s78fKQ2vVP +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.dX1iA91hD9 +++++ cat /tmp/tmp.s78fKQ2vVP +++++ rm /tmp/tmp.dX1iA91hD9 /tmp/tmp.s78fKQ2vVP +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-haproxy ++++ return +++ local cluster_proxy=some-name-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uYLXua7tYi +++ mktemp ++ local LAST_ERR=/tmp/tmp.pH6rndMesy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uYLXua7tYi ++ cat /tmp/tmp.pH6rndMesy ++ rm /tmp/tmp.uYLXua7tYi /tmp/tmp.pH6rndMesy ++ return 0 + [[ 3 == \3 ]] + patch_secret my-cluster-secrets monitor dGVzdC1wYXNzd29yZDI= + local secret=my-cluster-secrets + local key=monitor + local value=dGVzdC1wYXNzd29yZDI= + kubectl_bin patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}' ++ mktemp + local LAST_OUT=/tmp/tmp.otZF8ThaJq ++ mktemp + local LAST_ERR=/tmp/tmp.s0giednQDM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets '-p={"data":{"monitor": "dGVzdC1wYXNzd29yZDI="}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.otZF8ThaJq secret/my-cluster-secrets patched + cat /tmp/tmp.s0giednQDM + rm /tmp/tmp.otZF8ThaJq /tmp/tmp.s0giednQDM + return 0 + sleep 15 + wait_cluster_consistency some-name 3 3 + local cluster_name=some-name + local cluster_size=3 + local proxy_size=3 + '[' -z 3 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rYo7VVJidI +++ mktemp ++ local LAST_ERR=/tmp/tmp.rrh523qMAD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rYo7VVJidI ++ cat /tmp/tmp.rrh523qMAD ++ rm /tmp/tmp.rYo7VVJidI /tmp/tmp.rrh523qMAD ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.C7p1VKHNx2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RbxFAP4cq0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.C7p1VKHNx2 ++ cat /tmp/tmp.RbxFAP4cq0 ++ rm /tmp/tmp.C7p1VKHNx2 /tmp/tmp.RbxFAP4cq0 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9ubY9Y074h +++ mktemp ++ local LAST_ERR=/tmp/tmp.shvYn0mSq6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9ubY9Y074h ++ cat /tmp/tmp.shvYn0mSq6 ++ rm /tmp/tmp.9ubY9Y074h /tmp/tmp.shvYn0mSq6 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CZc3djiHp3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.d3iYzdtiWP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CZc3djiHp3 ++ cat /tmp/tmp.d3iYzdtiWP ++ rm /tmp/tmp.CZc3djiHp3 /tmp/tmp.d3iYzdtiWP ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lBvx6KThqV +++ mktemp ++ local LAST_ERR=/tmp/tmp.D3pUi9PUZw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lBvx6KThqV ++ cat /tmp/tmp.D3pUi9PUZw ++ rm /tmp/tmp.lBvx6KThqV /tmp/tmp.D3pUi9PUZw ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.yN66eK1Wen ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.3yIZv0KGsO +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.yN66eK1Wen +++++ cat /tmp/tmp.3yIZv0KGsO +++++ rm /tmp/tmp.yN66eK1Wen /tmp/tmp.3yIZv0KGsO +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-haproxy ++++ return +++ local cluster_proxy=some-name-haproxy +++ echo haproxy ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.D4xBN8iHzB +++ mktemp ++ local LAST_ERR=/tmp/tmp.GcezAwBEAZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.D4xBN8iHzB ++ cat /tmp/tmp.GcezAwBEAZ ++ rm /tmp/tmp.D4xBN8iHzB /tmp/tmp.GcezAwBEAZ ++ return 0 + [[ 3 == \3 ]] + compare_mysql_cmd select-3 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\''' + local command_id=select-3 + local 'command=SHOW DATABASES;' + local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc5.7 =~ 8\.0 ]] + run_mysql 'SHOW DATABASES;' '-h some-name-haproxy -umonitor -p'\''test-password2'\''' + local 'command=SHOW DATABASES;' + local 'uri=-h some-name-haproxy -umonitor -p'\''test-password2'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.R0ckEp7GkW +++ mktemp ++ local LAST_ERR=/tmp/tmp.J1ZmvpnMJu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.R0ckEp7GkW ++ cat /tmp/tmp.J1ZmvpnMJu ++ rm /tmp/tmp.R0ckEp7GkW /tmp/tmp.J1ZmvpnMJu ++ return 0 + client_pod=pxc-client-64b479df95-mlsmx + wait_pod pxc-client-64b479df95-mlsmx + local pod=pxc-client-64b479df95-mlsmx + local max_retry=480 + local ns= ++ echo pxc-client-64b479df95-mlsmx ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-64b479df95-mlsmx condition met pxc-client-64b479df95-mlsmx.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.q8i5quND1N/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1747/e2e-tests/users/compare/select-3.sql /tmp/tmp.q8i5quND1N/select-3.sql + destroy users-19584 + local namespace=users-19584 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v level=info + grep -v 'the object has been modified' + grep -v 'get backup status: Job.batch' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + tee /tmp/tmp.q8i5quND1N/operator.log ++ get_operator_pod + sort -u ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.xxpkMHvM8M +++ mktemp ++ local LAST_ERR=/tmp/tmp.5BtCY26emE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xxpkMHvM8M ++ cat /tmp/tmp.5BtCY26emE ++ rm /tmp/tmp.xxpkMHvM8M /tmp/tmp.5BtCY26emE ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-598ffb648b-hkvvk ++ mktemp + local LAST_OUT=/tmp/tmp.S69Zvavh7u ++ mktemp + local LAST_ERR=/tmp/tmp.Myeq8aX55V + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-598ffb648b-hkvvk + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.S69Zvavh7u + cat /tmp/tmp.Myeq8aX55V + rm /tmp/tmp.S69Zvavh7u /tmp/tmp.Myeq8aX55V + return 0 2024-07-03T15:00:29.901Z INFO setup Manager starting up {"gitCommit": "942809ab7fdb169936d1e3c60ac420171180c514", "gitBranch": "PR-1747-942809ab", "buildTime": "2024-07-03T12:54:51Z", "goVersion": "go1.22.5", "os": "linux", "arch": "amd64"} 2024-07-03T15:00:29.901Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.26.15-gke.1469000"} 2024-07-03T15:00:29.902Z INFO setup Registering Components. 2024-07-03T15:00:31.875Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2024-07-03T15:00:31.878Z INFO controller-runtime.metrics Starting metrics server 2024-07-03T15:00:31.878Z INFO setup Starting the Cmd. 2024-07-03T15:00:31.879Z INFO controller-runtime.certwatcher Updated current TLS certificate 2024-07-03T15:00:31.879Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2024-07-03T15:00:31.879Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2024-07-03T15:00:31.879Z INFO controller-runtime.webhook Starting webhook server 2024-07-03T15:00:31.879Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2024-07-03T15:00:31.880Z INFO controller-runtime.certwatcher Starting certificate watcher 2024-07-03T15:00:31.980Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2024-07-03T15:00:31.998Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2024-07-03T15:00:31.999Z DEBUG events percona-xtradb-cluster-operator-598ffb648b-hkvvk_db32acce-c96b-41ca-a6e1-1aa8aeefb316 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"e05708db-d8de-4860-be1b-757b5152826f","apiVersion":"coordination.k8s.io/v1","resourceVersion":"68157"}, "reason": "LeaderElection"} 2024-07-03T15:00:31.999Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2024-07-03T15:00:31.999Z INFO Starting Controller {"controller": "pxc-controller"} 2024-07-03T15:00:31.999Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2024-07-03T15:00:31.999Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2024-07-03T15:00:31.999Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"} 2024-07-03T15:00:31.999Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2024-07-03T15:00:32.127Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2024-07-03T15:00:32.127Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2024-07-03T15:00:32.127Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2024-07-03T15:01:03.156Z INFO Set CR version {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "db5512bb-ab80-4d99-bf93-44b1b8f1377a", "version": "1.15.0"} 2024-07-03T15:01:03.604Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "db5512bb-ab80-4d99-bf93-44b1b8f1377a", "object": "some-name-pxc"} 2024-07-03T15:01:03.732Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "db5512bb-ab80-4d99-bf93-44b1b8f1377a", "object": "some-name-pxc"} 2024-07-03T15:01:03.814Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "db5512bb-ab80-4d99-bf93-44b1b8f1377a", "object": "some-name-pxc"} 2024-07-03T15:01:03.934Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "db5512bb-ab80-4d99-bf93-44b1b8f1377a", "object": "some-name-pxc-unready"} 2024-07-03T15:01:04.093Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "db5512bb-ab80-4d99-bf93-44b1b8f1377a", "object": "some-name-proxysql"} 2024-07-03T15:01:04.222Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "db5512bb-ab80-4d99-bf93-44b1b8f1377a", "object": "some-name-proxysql"} 2024-07-03T15:01:04.352Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "db5512bb-ab80-4d99-bf93-44b1b8f1377a", "object": "some-name-proxysql-unready"} 2024-07-03T15:01:04.708Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ccb3469a-e238-4716-84e4-9fdebcf3ae1c", "object": "some-name-pxc"} 2024-07-03T15:01:04.863Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ccb3469a-e238-4716-84e4-9fdebcf3ae1c", "object": "some-name-pxc"} 2024-07-03T15:01:04.896Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ccb3469a-e238-4716-84e4-9fdebcf3ae1c", "object": "some-name-proxysql"} 2024-07-03T15:01:04.915Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ccb3469a-e238-4716-84e4-9fdebcf3ae1c", "object": "some-name-pxc"} 2024-07-03T15:01:05.002Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "13dc1f3a-6047-467e-aa8a-6964f842f654", "object": "some-name-pxc"} 2024-07-03T15:01:05.061Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "13dc1f3a-6047-467e-aa8a-6964f842f654", "object": "some-name-pxc"} 2024-07-03T15:01:09.700Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "aa27e1ce-e57b-4fc9-a1ac-4dce2c44b9e5", "object": "some-name-pxc"} 2024-07-03T15:01:09.789Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "aa27e1ce-e57b-4fc9-a1ac-4dce2c44b9e5", "object": "some-name-pxc"} 2024-07-03T15:01:14.930Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6163f44f-c1ac-431d-b87b-9543b4e20f64", "object": "some-name-pxc"} 2024-07-03T15:01:15.003Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6163f44f-c1ac-431d-b87b-9543b4e20f64", "object": "some-name-pxc"} 2024-07-03T15:01:20.081Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c3054071-fd2c-4d3b-9585-5f68f161b1ed", "object": "some-name-pxc"} 2024-07-03T15:01:20.135Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c3054071-fd2c-4d3b-9585-5f68f161b1ed", "object": "some-name-pxc"} 2024-07-03T15:01:25.249Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a701599a-b8f7-40c0-90b4-85a5e46a93e2", "object": "some-name-pxc"} 2024-07-03T15:01:25.350Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a701599a-b8f7-40c0-90b4-85a5e46a93e2", "object": "some-name-pxc"} 2024-07-03T15:01:25.543Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a17f4dc2-5edf-46a4-9ea7-c9ebfa0c8ae8", "object": "some-name-pxc"} 2024-07-03T15:01:25.652Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a17f4dc2-5edf-46a4-9ea7-c9ebfa0c8ae8", "object": "some-name-pxc"} 2024-07-03T15:01:30.524Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e5e572d5-2c8d-4635-81c8-08299f5cd574", "object": "some-name-pxc"} 2024-07-03T15:01:30.580Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e5e572d5-2c8d-4635-81c8-08299f5cd574", "object": "some-name-pxc"} 2024-07-03T15:01:35.715Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9c667635-f14d-4d38-b062-cb3a9adf778d", "object": "some-name-pxc"} 2024-07-03T15:01:35.786Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9c667635-f14d-4d38-b062-cb3a9adf778d", "object": "some-name-pxc"} 2024-07-03T15:01:35.993Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4c321359-f8c3-4c64-91ab-0541bcda2eb4", "object": "some-name-pxc"} 2024-07-03T15:01:36.068Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4c321359-f8c3-4c64-91ab-0541bcda2eb4", "object": "some-name-pxc"} 2024-07-03T15:01:40.986Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c43ebd98-6461-45b1-9a5c-b6026039a151", "object": "some-name-pxc"} 2024-07-03T15:01:41.062Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c43ebd98-6461-45b1-9a5c-b6026039a151", "object": "some-name-pxc"} 2024-07-03T15:01:46.180Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "650e0877-280c-4529-befa-38c5cc075e58", "object": "some-name-pxc"} 2024-07-03T15:01:46.249Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "650e0877-280c-4529-befa-38c5cc075e58", "object": "some-name-pxc"} 2024-07-03T15:01:51.353Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a1a20dfe-7640-4132-924f-fd0fc1abc8b4", "object": "some-name-pxc"} 2024-07-03T15:01:51.460Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a1a20dfe-7640-4132-924f-fd0fc1abc8b4", "object": "some-name-pxc"} 2024-07-03T15:01:56.712Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d919f873-75e5-4e5d-bb0a-8c3205ef8392", "object": "some-name-pxc"} 2024-07-03T15:01:57.184Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d919f873-75e5-4e5d-bb0a-8c3205ef8392", "object": "some-name-pxc"} 2024-07-03T15:02:02.344Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "68e550f4-db2b-46e0-a753-3b38dcd51b28", "object": "some-name-pxc"} 2024-07-03T15:02:02.437Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "68e550f4-db2b-46e0-a753-3b38dcd51b28", "object": "some-name-pxc"} 2024-07-03T15:02:07.581Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6f42c0a8-7e49-4b88-9196-035e5dfbfa31", "object": "some-name-pxc"} 2024-07-03T15:02:07.652Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6f42c0a8-7e49-4b88-9196-035e5dfbfa31", "object": "some-name-pxc"} 2024-07-03T15:02:12.751Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9476841e-b24b-4636-88f7-fad9aa4f7066", "object": "some-name-pxc"} 2024-07-03T15:02:12.817Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9476841e-b24b-4636-88f7-fad9aa4f7066", "object": "some-name-pxc"} 2024-07-03T15:02:17.926Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e30987f6-f8b9-453d-a528-d365f32a455a", "object": "some-name-pxc"} 2024-07-03T15:02:17.991Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e30987f6-f8b9-453d-a528-d365f32a455a", "object": "some-name-pxc"} 2024-07-03T15:02:18.220Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5a8f1106-6ef6-44d4-9b49-5966c3f635fd", "user": "operator"} 2024-07-03T15:02:18.266Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5a8f1106-6ef6-44d4-9b49-5966c3f635fd", "user": "monitor"} 2024-07-03T15:02:18.314Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5a8f1106-6ef6-44d4-9b49-5966c3f635fd"} 2024-07-03T15:02:18.372Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5a8f1106-6ef6-44d4-9b49-5966c3f635fd", "user": "xtrabackup"} 2024-07-03T15:02:18.430Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5a8f1106-6ef6-44d4-9b49-5966c3f635fd"} 2024-07-03T15:02:18.433Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5a8f1106-6ef6-44d4-9b49-5966c3f635fd", "object": "some-name-pxc"} 2024-07-03T15:02:18.516Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5a8f1106-6ef6-44d4-9b49-5966c3f635fd", "object": "some-name-pxc"} 2024-07-03T15:02:18.552Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5a8f1106-6ef6-44d4-9b49-5966c3f635fd", "err": "get primary pxc pod: not found"} 2024-07-03T15:02:23.188Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b6005628-5655-4968-b808-ff0e489d11e8", "object": "some-name-pxc"} 2024-07-03T15:02:23.281Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b6005628-5655-4968-b808-ff0e489d11e8", "object": "some-name-pxc"} 2024-07-03T15:02:23.361Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b6005628-5655-4968-b808-ff0e489d11e8", "err": "get primary pxc pod: not found"} 2024-07-03T15:02:28.493Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fbf62998-78d6-4665-b088-b1f2e9fec952", "object": "some-name-pxc"} 2024-07-03T15:02:28.586Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fbf62998-78d6-4665-b088-b1f2e9fec952", "object": "some-name-pxc"} 2024-07-03T15:02:28.647Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fbf62998-78d6-4665-b088-b1f2e9fec952", "err": "get primary pxc pod: not found"} 2024-07-03T15:02:33.765Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9972712a-2c70-4930-8192-3c0425d7aaff", "object": "some-name-pxc"} 2024-07-03T15:02:33.838Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9972712a-2c70-4930-8192-3c0425d7aaff", "object": "some-name-pxc"} 2024-07-03T15:02:33.886Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9972712a-2c70-4930-8192-3c0425d7aaff", "err": "get primary pxc pod: not found"} 2024-07-03T15:02:39.059Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "269d0df9-d3a5-4924-ab47-3b86682bc906", "object": "some-name-pxc"} 2024-07-03T15:02:39.124Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "269d0df9-d3a5-4924-ab47-3b86682bc906", "object": "some-name-pxc"} 2024-07-03T15:02:44.302Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4a9df919-828e-4d19-9ea4-96cd2726f6b9", "object": "some-name-pxc"} 2024-07-03T15:02:44.386Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4a9df919-828e-4d19-9ea4-96cd2726f6b9", "object": "some-name-pxc"} 2024-07-03T15:02:49.807Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7b5c9211-9db9-4c4c-83d9-72f5a294afa0", "object": "some-name-pxc"} 2024-07-03T15:02:49.869Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7b5c9211-9db9-4c4c-83d9-72f5a294afa0", "object": "some-name-pxc"} 2024-07-03T15:02:55.070Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b7add6a4-b410-4432-a76f-1eee03a6068e", "object": "some-name-pxc"} 2024-07-03T15:02:55.137Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b7add6a4-b410-4432-a76f-1eee03a6068e", "object": "some-name-pxc"} 2024-07-03T15:03:00.305Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "33276535-b107-4e76-9a79-e363d23bffa4", "object": "some-name-pxc"} 2024-07-03T15:03:00.371Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "33276535-b107-4e76-9a79-e363d23bffa4", "object": "some-name-pxc"} 2024-07-03T15:03:05.596Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7a30c096-e49b-4094-8ed7-da6cc0411fb3", "object": "some-name-pxc"} 2024-07-03T15:03:05.677Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7a30c096-e49b-4094-8ed7-da6cc0411fb3", "object": "some-name-pxc"} 2024-07-03T15:03:10.838Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f524ef20-5df6-4e63-af11-6ffc851058b4", "object": "some-name-pxc"} 2024-07-03T15:03:10.892Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f524ef20-5df6-4e63-af11-6ffc851058b4", "object": "some-name-pxc"} 2024-07-03T15:03:16.774Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ea5012e7-70ab-48c0-88a3-22df4680f71e", "object": "some-name-pxc"} 2024-07-03T15:03:16.833Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ea5012e7-70ab-48c0-88a3-22df4680f71e", "object": "some-name-pxc"} 2024-07-03T15:03:22.001Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6106ec66-a336-4bf8-a05c-1110c1dbcd03", "object": "some-name-pxc"} 2024-07-03T15:03:22.055Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6106ec66-a336-4bf8-a05c-1110c1dbcd03", "object": "some-name-pxc"} 2024-07-03T15:03:27.220Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ec3a7b8c-f667-4baf-a68e-175959c517e7", "object": "some-name-pxc"} 2024-07-03T15:03:27.274Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ec3a7b8c-f667-4baf-a68e-175959c517e7", "object": "some-name-pxc"} 2024-07-03T15:03:32.428Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "062f5765-7ca4-49a7-a8fd-aa35fab8acfe", "object": "some-name-pxc"} 2024-07-03T15:03:32.479Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "062f5765-7ca4-49a7-a8fd-aa35fab8acfe", "object": "some-name-pxc"} 2024-07-03T15:03:32.672Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6558a4b0-c16e-48d2-b8e9-09b0c95564bb", "object": "some-name-pxc"} 2024-07-03T15:03:32.733Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6558a4b0-c16e-48d2-b8e9-09b0c95564bb", "object": "some-name-pxc"} 2024-07-03T15:03:37.672Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "56abff33-cf46-4201-bd24-49e13c15f31b", "object": "some-name-pxc"} 2024-07-03T15:03:37.728Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "56abff33-cf46-4201-bd24-49e13c15f31b", "object": "some-name-pxc"} 2024-07-03T15:03:42.930Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "69d74c9b-d0f5-4abe-b9c3-c369fc20daff", "object": "some-name-pxc"} 2024-07-03T15:03:43.027Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "69d74c9b-d0f5-4abe-b9c3-c369fc20daff", "object": "some-name-pxc"} 2024-07-03T15:03:48.267Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3393c0a1-1888-4e4c-85f6-d86e20201b09", "object": "some-name-pxc"} 2024-07-03T15:03:48.329Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3393c0a1-1888-4e4c-85f6-d86e20201b09", "object": "some-name-pxc"} 2024-07-03T15:03:53.503Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d9988042-0181-42d4-8d43-a963a13019fb", "object": "some-name-pxc"} 2024-07-03T15:03:53.576Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d9988042-0181-42d4-8d43-a963a13019fb", "object": "some-name-pxc"} 2024-07-03T15:03:58.790Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d9ae1848-4bcc-4395-9b50-5b1b46182c83", "object": "some-name-pxc"} 2024-07-03T15:03:58.848Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d9ae1848-4bcc-4395-9b50-5b1b46182c83", "object": "some-name-pxc"} 2024-07-03T15:04:04.046Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5c882220-df87-4187-95bf-7afc8c2f7801", "object": "some-name-pxc"} 2024-07-03T15:04:04.112Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5c882220-df87-4187-95bf-7afc8c2f7801", "object": "some-name-pxc"} 2024-07-03T15:04:09.281Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2a8ef9c0-911c-4d69-a20e-830405be53f6", "object": "some-name-pxc"} 2024-07-03T15:04:09.340Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2a8ef9c0-911c-4d69-a20e-830405be53f6", "object": "some-name-pxc"} 2024-07-03T15:04:14.524Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ea0dca10-f155-4cf8-98a0-412de8ddccdb", "object": "some-name-pxc"} 2024-07-03T15:04:14.594Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ea0dca10-f155-4cf8-98a0-412de8ddccdb", "object": "some-name-pxc"} 2024-07-03T15:04:19.832Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cceff9b1-b3e9-4507-8e2d-2088b725d5e1", "object": "some-name-pxc"} 2024-07-03T15:04:19.902Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cceff9b1-b3e9-4507-8e2d-2088b725d5e1", "object": "some-name-pxc"} 2024-07-03T15:04:25.139Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "61c7a851-c39c-4c7c-a5ca-19f3c733d921", "object": "some-name-pxc"} 2024-07-03T15:04:25.253Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "61c7a851-c39c-4c7c-a5ca-19f3c733d921", "object": "some-name-pxc"} 2024-07-03T15:04:30.515Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7370e44b-9cf5-4183-9a7a-a5d6488076c1", "object": "some-name-pxc"} 2024-07-03T15:04:30.566Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7370e44b-9cf5-4183-9a7a-a5d6488076c1", "object": "some-name-pxc"} 2024-07-03T15:04:35.758Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5053aa76-54f5-42fe-a3ff-cb8536478074", "object": "some-name-pxc"} 2024-07-03T15:04:35.836Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5053aa76-54f5-42fe-a3ff-cb8536478074", "object": "some-name-pxc"} 2024-07-03T15:04:41.073Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "96f59fc2-9644-4f9e-9d9c-bbda83153c36", "object": "some-name-pxc"} 2024-07-03T15:04:41.273Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "96f59fc2-9644-4f9e-9d9c-bbda83153c36", "object": "some-name-pxc"} 2024-07-03T15:04:46.513Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "963e4b17-8a4a-4380-86fe-a30217dd55a2", "object": "some-name-pxc"} 2024-07-03T15:04:46.615Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "963e4b17-8a4a-4380-86fe-a30217dd55a2", "object": "some-name-pxc"} 2024-07-03T15:04:47.060Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "15dac28d-76df-426b-9a54-f9c7c380a401", "user": "root"} 2024-07-03T15:04:47.102Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "15dac28d-76df-426b-9a54-f9c7c380a401", "user": "replication"} 2024-07-03T15:04:47.104Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "15dac28d-76df-426b-9a54-f9c7c380a401", "object": "some-name-pxc"} 2024-07-03T15:04:47.167Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "15dac28d-76df-426b-9a54-f9c7c380a401", "object": "some-name-pxc"} 2024-07-03T15:04:47.250Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "15dac28d-76df-426b-9a54-f9c7c380a401", "new version": "5.7.44-48-57"} 2024-07-03T15:04:48.057Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "79af60af-9b02-47f6-842f-61139e11dd68", "object": "some-name-pxc"} 2024-07-03T15:04:48.114Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "79af60af-9b02-47f6-842f-61139e11dd68", "object": "some-name-pxc"} 2024-07-03T15:04:50.674Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "15dac28d-76df-426b-9a54-f9c7c380a401"} 2024-07-03T15:04:51.979Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "aa10154e-fc64-4647-891f-25aa3bf128a9", "object": "some-name-pxc"} 2024-07-03T15:04:52.061Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "aa10154e-fc64-4647-891f-25aa3bf128a9", "object": "some-name-pxc"} 2024-07-03T15:04:55.548Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "aa10154e-fc64-4647-891f-25aa3bf128a9"} 2024-07-03T15:04:57.226Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2b618e93-4e6e-4037-a08a-885f3f82fd9e", "object": "some-name-pxc"} 2024-07-03T15:04:57.284Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2b618e93-4e6e-4037-a08a-885f3f82fd9e", "object": "some-name-pxc"} 2024-07-03T15:05:00.980Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2b618e93-4e6e-4037-a08a-885f3f82fd9e"} 2024-07-03T15:05:02.657Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "58d4a444-3eba-409f-8d88-213e5389ce87", "object": "some-name-pxc"} 2024-07-03T15:05:02.722Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "58d4a444-3eba-409f-8d88-213e5389ce87", "object": "some-name-pxc"} 2024-07-03T15:05:06.258Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "58d4a444-3eba-409f-8d88-213e5389ce87"} 2024-07-03T15:05:07.911Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07afd489-8a7b-4a62-b272-a15c1de2a2c1", "object": "some-name-pxc"} 2024-07-03T15:05:07.990Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07afd489-8a7b-4a62-b272-a15c1de2a2c1", "object": "some-name-pxc"} 2024-07-03T15:05:11.463Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07afd489-8a7b-4a62-b272-a15c1de2a2c1"} 2024-07-03T15:05:13.224Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "abf7a65b-135d-4506-8967-dc97f9e81bda", "object": "some-name-pxc"} 2024-07-03T15:05:13.282Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "abf7a65b-135d-4506-8967-dc97f9e81bda", "object": "some-name-pxc"} 2024-07-03T15:05:16.683Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "abf7a65b-135d-4506-8967-dc97f9e81bda"} 2024-07-03T15:05:18.478Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "49117cfc-7dbd-42ef-b139-3b05180c981d", "object": "some-name-pxc"} 2024-07-03T15:05:18.602Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "49117cfc-7dbd-42ef-b139-3b05180c981d", "object": "some-name-pxc"} 2024-07-03T15:05:22.035Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "49117cfc-7dbd-42ef-b139-3b05180c981d"} 2024-07-03T15:05:23.805Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "22dd508f-8904-4df8-a7bc-90981059600d", "object": "some-name-pxc"} 2024-07-03T15:05:23.873Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "22dd508f-8904-4df8-a7bc-90981059600d", "object": "some-name-pxc"} 2024-07-03T15:05:28.475Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "22dd508f-8904-4df8-a7bc-90981059600d"} 2024-07-03T15:05:29.084Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "813db460-3d72-4623-9d70-e92066d86cad", "object": "some-name-pxc"} 2024-07-03T15:05:29.153Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "813db460-3d72-4623-9d70-e92066d86cad", "object": "some-name-pxc"} 2024-07-03T15:05:32.881Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "813db460-3d72-4623-9d70-e92066d86cad"} 2024-07-03T15:05:34.318Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1998ff27-ea2d-4ea4-bb23-45af5ccace1c", "object": "some-name-pxc"} 2024-07-03T15:05:34.394Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1998ff27-ea2d-4ea4-bb23-45af5ccace1c", "object": "some-name-pxc"} 2024-07-03T15:05:37.937Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1998ff27-ea2d-4ea4-bb23-45af5ccace1c"} 2024-07-03T15:05:39.676Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a0c9bb32-faba-4f6d-a8fb-7ca548ca5fce", "object": "some-name-pxc"} 2024-07-03T15:05:39.761Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a0c9bb32-faba-4f6d-a8fb-7ca548ca5fce", "object": "some-name-pxc"} 2024-07-03T15:05:43.267Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a0c9bb32-faba-4f6d-a8fb-7ca548ca5fce"} 2024-07-03T15:05:44.943Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "73cf71b4-aa95-4216-a6a8-1b7aac1d389f", "object": "some-name-pxc"} 2024-07-03T15:05:45.021Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "73cf71b4-aa95-4216-a6a8-1b7aac1d389f", "object": "some-name-pxc"} 2024-07-03T15:05:48.578Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "73cf71b4-aa95-4216-a6a8-1b7aac1d389f"} 2024-07-03T15:05:50.298Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f1b439f4-4b91-49ad-a6d0-345707e8a7ba", "user": "root"} 2024-07-03T15:05:50.328Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f1b439f4-4b91-49ad-a6d0-345707e8a7ba", "user": "root"} 2024-07-03T15:05:50.337Z INFO MySQL init secret created {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f1b439f4-4b91-49ad-a6d0-345707e8a7ba", "secret": "some-name-mysql-init", "user": "root"} 2024-07-03T15:05:56.012Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f1b439f4-4b91-49ad-a6d0-345707e8a7ba"} 2024-07-03T15:05:56.029Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f1b439f4-4b91-49ad-a6d0-345707e8a7ba", "user": "root"} 2024-07-03T15:05:56.031Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f1b439f4-4b91-49ad-a6d0-345707e8a7ba", "object": "some-name-pxc"} 2024-07-03T15:05:56.484Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f1b439f4-4b91-49ad-a6d0-345707e8a7ba", "object": "some-name-pxc"} 2024-07-03T15:06:00.659Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f1b439f4-4b91-49ad-a6d0-345707e8a7ba"} 2024-07-03T15:06:02.207Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f6b72c73-e80f-4dad-acec-26e314cfa306", "object": "some-name-pxc"} 2024-07-03T15:06:02.349Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f6b72c73-e80f-4dad-acec-26e314cfa306", "object": "some-name-pxc"} 2024-07-03T15:06:05.870Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f6b72c73-e80f-4dad-acec-26e314cfa306"} 2024-07-03T15:06:07.541Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f91fe035-d49b-4b21-97a1-c4ec6b0edf19", "object": "some-name-pxc"} 2024-07-03T15:06:07.682Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f91fe035-d49b-4b21-97a1-c4ec6b0edf19", "object": "some-name-pxc"} 2024-07-03T15:06:09.491Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f5c3d8ff-6a4f-40f2-8801-867e0edde957", "object": "some-name-pxc"} 2024-07-03T15:06:09.599Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f5c3d8ff-6a4f-40f2-8801-867e0edde957", "object": "some-name-pxc"} 2024-07-03T15:06:09.676Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f5c3d8ff-6a4f-40f2-8801-867e0edde957", "object": "some-name-proxysql"} 2024-07-03T15:06:10.085Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "25bb079b-8693-4ce3-9d58-fa7b54986f6b", "object": "some-name-pxc"} 2024-07-03T15:06:10.279Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "25bb079b-8693-4ce3-9d58-fa7b54986f6b", "object": "some-name-pxc"} 2024-07-03T15:06:11.391Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f91fe035-d49b-4b21-97a1-c4ec6b0edf19"} 2024-07-03T15:06:12.888Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "daf62e1f-f1d9-4eab-90e5-f1d80649e7d4", "object": "some-name-pxc"} 2024-07-03T15:06:12.954Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "daf62e1f-f1d9-4eab-90e5-f1d80649e7d4", "object": "some-name-pxc"} 2024-07-03T15:06:18.148Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c8a7f609-3715-4b73-9206-5522d6162b49", "object": "some-name-pxc"} 2024-07-03T15:06:18.207Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c8a7f609-3715-4b73-9206-5522d6162b49", "object": "some-name-pxc"} 2024-07-03T15:06:23.376Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8077a0b4-8276-4acf-a705-bf0f1cab9c5d", "object": "some-name-pxc"} 2024-07-03T15:06:23.437Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8077a0b4-8276-4acf-a705-bf0f1cab9c5d", "object": "some-name-pxc"} 2024-07-03T15:06:28.639Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a0efda07-e0b9-438d-b8c7-e8c74272ff2e", "object": "some-name-pxc"} 2024-07-03T15:06:28.694Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a0efda07-e0b9-438d-b8c7-e8c74272ff2e", "object": "some-name-pxc"} 2024-07-03T15:06:28.954Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "35bfbb47-eef1-4f3a-abff-8cdccdf90272", "object": "some-name-pxc"} 2024-07-03T15:06:29.018Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "35bfbb47-eef1-4f3a-abff-8cdccdf90272", "object": "some-name-pxc"} 2024-07-03T15:06:33.222Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "35bfbb47-eef1-4f3a-abff-8cdccdf90272", "error": "exec syncusers: command terminated with exit code 1 / / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-03T15:06:33.971Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9ad498f7-d70c-421c-a0a4-6158f153bc23", "object": "some-name-pxc"} 2024-07-03T15:06:34.076Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9ad498f7-d70c-421c-a0a4-6158f153bc23", "object": "some-name-pxc"} 2024-07-03T15:06:38.260Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9ad498f7-d70c-421c-a0a4-6158f153bc23", "error": "exec syncusers: command terminated with exit code 1 / / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR (line:512) : The cluster (with writer hostgroup:11) has not been configured in ProxySQL\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-03T15:06:39.301Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9cd555c2-726f-495a-b2d0-692da75f464e", "user": "proxyadmin"} 2024-07-03T15:06:39.301Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9cd555c2-726f-495a-b2d0-692da75f464e", "user": "proxyadmin"} 2024-07-03T15:06:39.395Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9cd555c2-726f-495a-b2d0-692da75f464e", "user": "proxyadmin"} 2024-07-03T15:06:39.405Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9cd555c2-726f-495a-b2d0-692da75f464e", "user": "proxyadmin"} 2024-07-03T15:06:39.405Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9cd555c2-726f-495a-b2d0-692da75f464e", "last-applied-secret": "1a55867d0f4fa0d0643653b2eb57b2f690315d8d843a10599c9839c2a779a95f"} 2024-07-03T15:06:39.408Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9cd555c2-726f-495a-b2d0-692da75f464e", "object": "some-name-pxc"} 2024-07-03T15:06:39.479Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9cd555c2-726f-495a-b2d0-692da75f464e", "object": "some-name-pxc"} 2024-07-03T15:06:39.522Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9cd555c2-726f-495a-b2d0-692da75f464e", "object": "some-name-proxysql"} 2024-07-03T15:06:39.651Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9cd555c2-726f-495a-b2d0-692da75f464e", "error": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-03T15:06:39.878Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d01b5af2-2137-40fb-bc5c-11b7ecf178a9", "object": "some-name-pxc"} 2024-07-03T15:06:39.974Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d01b5af2-2137-40fb-bc5c-11b7ecf178a9", "object": "some-name-pxc"} 2024-07-03T15:06:45.162Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "bf213949-8074-41b9-bc7f-ac33977776c8", "object": "some-name-pxc"} 2024-07-03T15:06:45.269Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "bf213949-8074-41b9-bc7f-ac33977776c8", "object": "some-name-pxc"} 2024-07-03T15:06:50.178Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a87eab98-28d8-40b0-bbbd-0ed0d9766344", "object": "some-name-pxc"} 2024-07-03T15:06:50.244Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a87eab98-28d8-40b0-bbbd-0ed0d9766344", "object": "some-name-pxc"} 2024-07-03T15:06:55.480Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f2f14098-1fb4-47a4-b4d8-aae610f31808", "object": "some-name-pxc"} 2024-07-03T15:06:55.646Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f2f14098-1fb4-47a4-b4d8-aae610f31808", "object": "some-name-pxc"} 2024-07-03T15:06:55.876Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "db08176c-b20f-4480-9ec6-9656e9335d0e", "object": "some-name-pxc"} 2024-07-03T15:06:55.984Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "db08176c-b20f-4480-9ec6-9656e9335d0e", "object": "some-name-pxc"} 2024-07-03T15:07:03.835Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2f1d5074-b732-415e-bbe4-515e19b9a367", "object": "some-name-pxc"} 2024-07-03T15:07:03.903Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2f1d5074-b732-415e-bbe4-515e19b9a367", "object": "some-name-pxc"} 2024-07-03T15:07:08.839Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "74057c03-2869-48eb-9848-43206f362c1f", "object": "some-name-pxc"} 2024-07-03T15:07:08.935Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "74057c03-2869-48eb-9848-43206f362c1f", "object": "some-name-pxc"} 2024-07-03T15:07:14.140Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "be133005-eed5-4503-97dc-131430cc3019", "object": "some-name-pxc"} 2024-07-03T15:07:14.203Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "be133005-eed5-4503-97dc-131430cc3019", "object": "some-name-pxc"} 2024-07-03T15:07:30.474Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6bcc0ed2-9294-4cb7-99c0-c7421e1b00d5", "object": "some-name-pxc"} 2024-07-03T15:07:30.546Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6bcc0ed2-9294-4cb7-99c0-c7421e1b00d5", "object": "some-name-pxc"} 2024-07-03T15:07:35.758Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8ea59a9b-6ea9-4a05-be48-341c6df4d578", "object": "some-name-pxc"} 2024-07-03T15:07:35.845Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8ea59a9b-6ea9-4a05-be48-341c6df4d578", "object": "some-name-pxc"} 2024-07-03T15:07:36.393Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6bcc0ed2-9294-4cb7-99c0-c7421e1b00d5"} 2024-07-03T15:07:40.756Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e1288fdc-ac9f-4a38-9a83-ae6bc951760e", "object": "some-name-pxc"} 2024-07-03T15:07:40.846Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e1288fdc-ac9f-4a38-9a83-ae6bc951760e", "object": "some-name-pxc"} 2024-07-03T15:07:46.291Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e1288fdc-ac9f-4a38-9a83-ae6bc951760e"} 2024-07-03T15:07:46.309Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "227023a6-4fdb-4a3f-b4b0-ac00ed054301", "object": "some-name-pxc"} 2024-07-03T15:07:46.389Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "227023a6-4fdb-4a3f-b4b0-ac00ed054301", "object": "some-name-pxc"} 2024-07-03T15:07:48.171Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cf46ac00-deeb-4cad-bdd7-f019b1135c31", "object": "some-name-pxc"} 2024-07-03T15:07:48.231Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cf46ac00-deeb-4cad-bdd7-f019b1135c31", "object": "some-name-pxc"} 2024-07-03T15:07:48.273Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cf46ac00-deeb-4cad-bdd7-f019b1135c31", "object": "some-name-proxysql"} 2024-07-03T15:07:48.640Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7c68e999-bd10-4293-a68d-90b7ed9a169a", "object": "some-name-pxc"} 2024-07-03T15:07:48.797Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7c68e999-bd10-4293-a68d-90b7ed9a169a", "object": "some-name-pxc"} 2024-07-03T15:07:51.649Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2465ad2c-e675-45b5-a977-8b7437a7548a", "object": "some-name-pxc"} 2024-07-03T15:07:51.736Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2465ad2c-e675-45b5-a977-8b7437a7548a", "object": "some-name-pxc"} 2024-07-03T15:07:51.982Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1117391f-c89b-4c76-8ee9-ce275a0a3fe3", "user": "xtrabackup"} 2024-07-03T15:07:52.003Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1117391f-c89b-4c76-8ee9-ce275a0a3fe3", "user": "xtrabackup"} 2024-07-03T15:07:52.023Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1117391f-c89b-4c76-8ee9-ce275a0a3fe3", "secret": "some-name-mysql-init", "user": "xtrabackup"} 2024-07-03T15:07:52.038Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1117391f-c89b-4c76-8ee9-ce275a0a3fe3", "user": "xtrabackup"} 2024-07-03T15:07:52.039Z INFO PXC pods will be restarted {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1117391f-c89b-4c76-8ee9-ce275a0a3fe3", "last-applied-secret": "d11bcc70ca53ea410cd455e13d82e7225917b0861d9b677ed34c02c5f6fbaf1b"} 2024-07-03T15:07:52.041Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1117391f-c89b-4c76-8ee9-ce275a0a3fe3", "object": "some-name-pxc"} 2024-07-03T15:07:52.120Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1117391f-c89b-4c76-8ee9-ce275a0a3fe3", "object": "some-name-pxc"} 2024-07-03T15:07:53.407Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f107a3eb-cbb8-4648-9d19-3f5d28de49b7", "object": "some-name-pxc"} 2024-07-03T15:07:53.481Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f107a3eb-cbb8-4648-9d19-3f5d28de49b7", "object": "some-name-pxc"} 2024-07-03T15:07:58.277Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1117391f-c89b-4c76-8ee9-ce275a0a3fe3"} 2024-07-03T15:07:58.759Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5f8af8c8-bb87-4b7b-9e12-b489757969c7", "object": "some-name-pxc"} 2024-07-03T15:07:58.820Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5f8af8c8-bb87-4b7b-9e12-b489757969c7", "object": "some-name-pxc"} 2024-07-03T15:07:59.404Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "28c9a48d-859e-4ea3-93f2-ed983140d2c1", "object": "some-name-pxc"} 2024-07-03T15:07:59.470Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "28c9a48d-859e-4ea3-93f2-ed983140d2c1", "object": "some-name-pxc"} 2024-07-03T15:08:03.765Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f4c518b2-a283-4ac8-969a-133ce1fbed58", "object": "some-name-pxc"} 2024-07-03T15:08:03.836Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f4c518b2-a283-4ac8-969a-133ce1fbed58", "object": "some-name-pxc"} 2024-07-03T15:08:09.091Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "605e84f6-e4b7-4ddb-8a48-46964229b082", "object": "some-name-pxc"} 2024-07-03T15:08:09.220Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "605e84f6-e4b7-4ddb-8a48-46964229b082", "object": "some-name-pxc"} 2024-07-03T15:08:14.358Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "dd9db195-2361-4a09-a182-fda8aa6e4b10", "object": "some-name-pxc"} 2024-07-03T15:08:14.458Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "dd9db195-2361-4a09-a182-fda8aa6e4b10", "object": "some-name-pxc"} 2024-07-03T15:08:19.675Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3dafbfea-5e22-4688-9c8e-3de7a172b0c1", "object": "some-name-pxc"} 2024-07-03T15:08:19.745Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3dafbfea-5e22-4688-9c8e-3de7a172b0c1", "object": "some-name-pxc"} 2024-07-03T15:08:24.959Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7b2cebe6-943c-497a-8d3f-06f41cffbfe9", "object": "some-name-pxc"} 2024-07-03T15:08:25.083Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7b2cebe6-943c-497a-8d3f-06f41cffbfe9", "object": "some-name-pxc"} 2024-07-03T15:08:30.355Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "50ec1c90-34ec-41b5-9b18-b84a66ec73ea", "object": "some-name-pxc"} 2024-07-03T15:08:30.426Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "50ec1c90-34ec-41b5-9b18-b84a66ec73ea", "object": "some-name-pxc"} 2024-07-03T15:08:35.642Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "80d17549-7699-412c-8b52-d0300708e0a6", "object": "some-name-pxc"} 2024-07-03T15:08:35.809Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "80d17549-7699-412c-8b52-d0300708e0a6", "object": "some-name-pxc"} 2024-07-03T15:08:41.002Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ba7a65b9-faea-4022-b333-ea799a58f81e", "object": "some-name-pxc"} 2024-07-03T15:08:41.148Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ba7a65b9-faea-4022-b333-ea799a58f81e", "object": "some-name-pxc"} 2024-07-03T15:08:46.547Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b01116f0-794b-49d5-9dc7-0affcc5090a9", "object": "some-name-pxc"} 2024-07-03T15:08:46.606Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b01116f0-794b-49d5-9dc7-0affcc5090a9", "object": "some-name-pxc"} 2024-07-03T15:08:51.769Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "45351a19-f0c9-4c50-90da-24e411a9d00e", "object": "some-name-pxc"} 2024-07-03T15:08:51.842Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "45351a19-f0c9-4c50-90da-24e411a9d00e", "object": "some-name-pxc"} 2024-07-03T15:08:52.092Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ad9ff94a-2e79-45a7-b27c-b4661af97e81", "object": "some-name-pxc"} 2024-07-03T15:08:52.174Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ad9ff94a-2e79-45a7-b27c-b4661af97e81", "object": "some-name-pxc"} 2024-07-03T15:08:57.066Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "563f3782-48d2-4ceb-98f0-4ba2c3a1ceba", "object": "some-name-pxc"} 2024-07-03T15:08:57.183Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "563f3782-48d2-4ceb-98f0-4ba2c3a1ceba", "object": "some-name-pxc"} 2024-07-03T15:09:02.460Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "0e6e3265-de27-42a5-9bc4-43ff0d7e7b71", "object": "some-name-pxc"} 2024-07-03T15:09:02.547Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "0e6e3265-de27-42a5-9bc4-43ff0d7e7b71", "object": "some-name-pxc"} 2024-07-03T15:09:07.710Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c17181ba-4504-4410-9d91-0821947115cc", "object": "some-name-pxc"} 2024-07-03T15:09:07.773Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c17181ba-4504-4410-9d91-0821947115cc", "object": "some-name-pxc"} 2024-07-03T15:09:12.976Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fd10e5a9-8f3e-43a6-b4af-bcaee9205f33", "object": "some-name-pxc"} 2024-07-03T15:09:13.032Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fd10e5a9-8f3e-43a6-b4af-bcaee9205f33", "object": "some-name-pxc"} 2024-07-03T15:09:18.186Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "70ad4dd2-9999-43fe-8b98-dff8b083f7aa", "object": "some-name-pxc"} 2024-07-03T15:09:18.259Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "70ad4dd2-9999-43fe-8b98-dff8b083f7aa", "object": "some-name-pxc"} 2024-07-03T15:09:23.419Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4409e1ef-ed3f-4b0d-abe1-b19ee540130b", "object": "some-name-pxc"} 2024-07-03T15:09:23.482Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4409e1ef-ed3f-4b0d-abe1-b19ee540130b", "object": "some-name-pxc"} 2024-07-03T15:09:28.626Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f75796bf-31ef-43e8-98b6-991f3e997195", "object": "some-name-pxc"} 2024-07-03T15:09:28.688Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f75796bf-31ef-43e8-98b6-991f3e997195", "object": "some-name-pxc"} 2024-07-03T15:09:33.856Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "17b7ca3d-dd5a-4431-871a-7c40c04e49a6", "object": "some-name-pxc"} 2024-07-03T15:09:33.926Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "17b7ca3d-dd5a-4431-871a-7c40c04e49a6", "object": "some-name-pxc"} 2024-07-03T15:09:39.117Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4bf1d729-3083-4ea8-8fd4-35e936b9bfa3", "object": "some-name-pxc"} 2024-07-03T15:09:39.168Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4bf1d729-3083-4ea8-8fd4-35e936b9bfa3", "object": "some-name-pxc"} 2024-07-03T15:09:39.374Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "0db75a96-ac3e-4889-bc69-e7d7e71e1b4f", "object": "some-name-pxc"} 2024-07-03T15:09:39.435Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "0db75a96-ac3e-4889-bc69-e7d7e71e1b4f", "object": "some-name-pxc"} 2024-07-03T15:09:44.373Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ffc5c7d7-e250-40b5-b201-2a9508274dfc", "object": "some-name-pxc"} 2024-07-03T15:09:44.435Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ffc5c7d7-e250-40b5-b201-2a9508274dfc", "object": "some-name-pxc"} 2024-07-03T15:09:49.709Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c7e5cb2f-31ab-4501-bde1-e78a217fe6a8", "object": "some-name-pxc"} 2024-07-03T15:09:49.769Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c7e5cb2f-31ab-4501-bde1-e78a217fe6a8", "object": "some-name-pxc"} 2024-07-03T15:09:49.821Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c7e5cb2f-31ab-4501-bde1-e78a217fe6a8", "err": "failed to connect to pod some-name-pxc-0: dial tcp 10.2.89.49:33062: connect: connection refused"} 2024-07-03T15:09:54.935Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d45170ba-9a49-48e8-b413-3d10449d4ad7", "object": "some-name-pxc"} 2024-07-03T15:09:54.999Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d45170ba-9a49-48e8-b413-3d10449d4ad7", "object": "some-name-pxc"} 2024-07-03T15:09:55.048Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d45170ba-9a49-48e8-b413-3d10449d4ad7", "primary name": "some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local"} 2024-07-03T15:10:00.137Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b8ed5172-39dd-47f5-a882-f364ea84ec8b", "object": "some-name-pxc"} 2024-07-03T15:10:00.221Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b8ed5172-39dd-47f5-a882-f364ea84ec8b", "object": "some-name-pxc"} 2024-07-03T15:10:00.272Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b8ed5172-39dd-47f5-a882-f364ea84ec8b", "primary name": "some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local"} 2024-07-03T15:10:05.387Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f7371fde-7bb3-496f-afd8-1c305b3b9213", "object": "some-name-pxc"} 2024-07-03T15:10:05.463Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f7371fde-7bb3-496f-afd8-1c305b3b9213", "object": "some-name-pxc"} 2024-07-03T15:10:05.517Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f7371fde-7bb3-496f-afd8-1c305b3b9213", "primary name": "some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local"} 2024-07-03T15:10:10.814Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "915191b3-28e6-4e61-9550-8a8601dc064c", "object": "some-name-pxc"} 2024-07-03T15:10:10.878Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "915191b3-28e6-4e61-9550-8a8601dc064c", "object": "some-name-pxc"} 2024-07-03T15:10:10.917Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "915191b3-28e6-4e61-9550-8a8601dc064c", "primary name": "some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local"} 2024-07-03T15:10:16.660Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6ff80958-7b2e-4dba-b145-b5d2891c160e", "object": "some-name-pxc"} 2024-07-03T15:10:16.723Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6ff80958-7b2e-4dba-b145-b5d2891c160e", "object": "some-name-pxc"} 2024-07-03T15:10:21.863Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "eb1c4c16-088e-4bbd-b9c4-3ecc1f04a9ea", "object": "some-name-pxc"} 2024-07-03T15:10:21.974Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "eb1c4c16-088e-4bbd-b9c4-3ecc1f04a9ea", "object": "some-name-pxc"} 2024-07-03T15:10:22.016Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "eb1c4c16-088e-4bbd-b9c4-3ecc1f04a9ea", "primary name": "some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local"} 2024-07-03T15:10:27.118Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "676a35a3-06f1-4c77-a62a-06acee3aa109", "object": "some-name-pxc"} 2024-07-03T15:10:27.178Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "676a35a3-06f1-4c77-a62a-06acee3aa109", "object": "some-name-pxc"} 2024-07-03T15:10:32.315Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c0774eae-2eb7-4945-a22d-d31bd18e2f6c", "object": "some-name-pxc"} 2024-07-03T15:10:32.370Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c0774eae-2eb7-4945-a22d-d31bd18e2f6c", "object": "some-name-pxc"} 2024-07-03T15:10:37.510Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "34002264-c180-43d2-97b4-faeee788a2a8", "object": "some-name-pxc"} 2024-07-03T15:10:37.576Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "34002264-c180-43d2-97b4-faeee788a2a8", "object": "some-name-pxc"} 2024-07-03T15:10:42.795Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "92701751-1098-4d38-ab7b-0ff2c14f70af", "object": "some-name-pxc"} 2024-07-03T15:10:42.865Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "92701751-1098-4d38-ab7b-0ff2c14f70af", "object": "some-name-pxc"} 2024-07-03T15:10:48.085Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "167991a4-8a64-4723-a245-348a92f725ba", "object": "some-name-pxc"} 2024-07-03T15:10:48.145Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "167991a4-8a64-4723-a245-348a92f725ba", "object": "some-name-pxc"} 2024-07-03T15:10:48.343Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3ef51bdb-cb0d-420b-94dc-af61d87a7208", "object": "some-name-pxc"} 2024-07-03T15:10:48.449Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3ef51bdb-cb0d-420b-94dc-af61d87a7208", "object": "some-name-pxc"} 2024-07-03T15:10:51.968Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3ef51bdb-cb0d-420b-94dc-af61d87a7208"} 2024-07-03T15:10:53.348Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "781d8db7-7228-49b3-8c0a-10ff88c8f758", "object": "some-name-pxc"} 2024-07-03T15:10:53.413Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "781d8db7-7228-49b3-8c0a-10ff88c8f758", "object": "some-name-pxc"} 2024-07-03T15:10:57.247Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "781d8db7-7228-49b3-8c0a-10ff88c8f758"} 2024-07-03T15:10:58.607Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a628e1e6-d8bd-4e7b-9a60-ebe7e266da9b", "object": "some-name-pxc"} 2024-07-03T15:10:58.686Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a628e1e6-d8bd-4e7b-9a60-ebe7e266da9b", "object": "some-name-pxc"} 2024-07-03T15:11:02.366Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a628e1e6-d8bd-4e7b-9a60-ebe7e266da9b"} 2024-07-03T15:11:03.935Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "be80e7e9-d44b-4fa7-a13f-408bdb509229", "object": "some-name-pxc"} 2024-07-03T15:11:04.005Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "be80e7e9-d44b-4fa7-a13f-408bdb509229", "object": "some-name-pxc"} 2024-07-03T15:11:07.559Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "be80e7e9-d44b-4fa7-a13f-408bdb509229"} 2024-07-03T15:11:09.197Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f33f52d7-a1d4-433b-bbe6-92b616fa7a78", "user": "monitor"} 2024-07-03T15:11:09.221Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f33f52d7-a1d4-433b-bbe6-92b616fa7a78", "user": "monitor"} 2024-07-03T15:11:09.234Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f33f52d7-a1d4-433b-bbe6-92b616fa7a78", "secret": "some-name-mysql-init", "user": "monitor"} 2024-07-03T15:11:09.286Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f33f52d7-a1d4-433b-bbe6-92b616fa7a78", "user": "monitor"} 2024-07-03T15:11:09.306Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f33f52d7-a1d4-433b-bbe6-92b616fa7a78", "user": "monitor"} 2024-07-03T15:11:09.306Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f33f52d7-a1d4-433b-bbe6-92b616fa7a78", "last-applied-secret": "4e3282050745046d5044e0fcb5f2bf140d7320a1256a79c8ca2184f38f4f3098"} 2024-07-03T15:11:09.309Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f33f52d7-a1d4-433b-bbe6-92b616fa7a78", "object": "some-name-pxc"} 2024-07-03T15:11:09.375Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f33f52d7-a1d4-433b-bbe6-92b616fa7a78", "object": "some-name-pxc"} 2024-07-03T15:11:09.475Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f33f52d7-a1d4-433b-bbe6-92b616fa7a78", "object": "some-name-proxysql"} 2024-07-03T15:11:09.821Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e41cdd7b-2b00-4fda-9b65-0fa4590fc248", "object": "some-name-pxc"} 2024-07-03T15:11:09.881Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e41cdd7b-2b00-4fda-9b65-0fa4590fc248", "object": "some-name-pxc"} 2024-07-03T15:11:12.342Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f33f52d7-a1d4-433b-bbe6-92b616fa7a78", "error": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") / / ", "errorVerbose": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") / / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-03T15:11:14.812Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cd7d65d9-3409-4129-84d1-ecf87c24594f", "object": "some-name-pxc"} 2024-07-03T15:11:14.868Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cd7d65d9-3409-4129-84d1-ecf87c24594f", "object": "some-name-pxc"} 2024-07-03T15:11:22.632Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c076dd69-bf82-4d92-ae92-7edcf08421a3", "object": "some-name-pxc"} 2024-07-03T15:11:22.757Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c076dd69-bf82-4d92-ae92-7edcf08421a3", "object": "some-name-pxc"} 2024-07-03T15:11:27.624Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "11fcf229-e257-4b39-8b21-b3eca87bd8c8", "object": "some-name-pxc"} 2024-07-03T15:11:27.780Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "11fcf229-e257-4b39-8b21-b3eca87bd8c8", "object": "some-name-pxc"} 2024-07-03T15:11:40.542Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "14647449-91fc-4bca-90dc-4f9edae42cdd", "object": "some-name-pxc"} 2024-07-03T15:11:40.595Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "14647449-91fc-4bca-90dc-4f9edae42cdd", "object": "some-name-pxc"} 2024-07-03T15:11:45.822Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "de2f3c19-f890-4920-a723-ca34915c6921", "object": "some-name-pxc"} 2024-07-03T15:11:45.879Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "de2f3c19-f890-4920-a723-ca34915c6921", "object": "some-name-pxc"} 2024-07-03T15:11:46.086Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d060a24a-6864-4662-9f2f-4ad5f2e5c81e", "object": "some-name-pxc"} 2024-07-03T15:11:46.158Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d060a24a-6864-4662-9f2f-4ad5f2e5c81e", "object": "some-name-pxc"} 2024-07-03T15:11:49.988Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d060a24a-6864-4662-9f2f-4ad5f2e5c81e"} 2024-07-03T15:11:51.091Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "68582711-4e9f-4532-a1ff-5d299d1235ae", "object": "some-name-pxc"} 2024-07-03T15:11:51.235Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "68582711-4e9f-4532-a1ff-5d299d1235ae", "object": "some-name-pxc"} 2024-07-03T15:11:54.899Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "68582711-4e9f-4532-a1ff-5d299d1235ae"} 2024-07-03T15:11:56.827Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3d0038d8-a997-4e1f-a3f7-06c4b4497e6e", "object": "some-name-pxc"} 2024-07-03T15:11:56.991Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3d0038d8-a997-4e1f-a3f7-06c4b4497e6e", "object": "some-name-pxc"} 2024-07-03T15:12:01.024Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3d0038d8-a997-4e1f-a3f7-06c4b4497e6e"} 2024-07-03T15:12:02.276Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7ebb52ef-2d8d-4649-a2f0-791c2c45b222", "object": "some-name-pxc"} 2024-07-03T15:12:02.341Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7ebb52ef-2d8d-4649-a2f0-791c2c45b222", "object": "some-name-pxc"} 2024-07-03T15:12:06.215Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7ebb52ef-2d8d-4649-a2f0-791c2c45b222"} 2024-07-03T15:12:07.556Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c16b3f86-c7af-4c20-9c7e-5fafe0dc2cef", "object": "some-name-pxc"} 2024-07-03T15:12:07.628Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c16b3f86-c7af-4c20-9c7e-5fafe0dc2cef", "object": "some-name-pxc"} 2024-07-03T15:12:11.470Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c16b3f86-c7af-4c20-9c7e-5fafe0dc2cef"} 2024-07-03T15:12:12.815Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "47a10d9d-cf69-46ef-bb96-52409cd38fa2", "user": "operator"} 2024-07-03T15:12:12.840Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "47a10d9d-cf69-46ef-bb96-52409cd38fa2", "user": "operator"} 2024-07-03T15:12:12.849Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "47a10d9d-cf69-46ef-bb96-52409cd38fa2", "secret": "some-name-mysql-init", "user": "operator"} 2024-07-03T15:12:12.861Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "47a10d9d-cf69-46ef-bb96-52409cd38fa2", "user": "operator"} 2024-07-03T15:12:12.862Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "47a10d9d-cf69-46ef-bb96-52409cd38fa2", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"} 2024-07-03T15:12:12.864Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "47a10d9d-cf69-46ef-bb96-52409cd38fa2", "object": "some-name-pxc"} 2024-07-03T15:12:12.927Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "47a10d9d-cf69-46ef-bb96-52409cd38fa2", "object": "some-name-pxc"} 2024-07-03T15:12:12.959Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "47a10d9d-cf69-46ef-bb96-52409cd38fa2", "object": "some-name-proxysql"} 2024-07-03T15:12:13.220Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "10c78b56-ebe4-4aec-8709-0bb7b22c6dfd", "object": "some-name-pxc"} 2024-07-03T15:12:13.287Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "10c78b56-ebe4-4aec-8709-0bb7b22c6dfd", "object": "some-name-pxc"} 2024-07-03T15:12:14.335Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "47a10d9d-cf69-46ef-bb96-52409cd38fa2", "error": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-19584.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-19584.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-19584.svc.' (using password: YES)\nERROR (line:569) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-19584.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-19584.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-19584.svc.' (using password: YES)\nERROR (line:569) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-03T15:12:18.278Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "526c4321-2a75-46fe-9869-997891b06973", "object": "some-name-pxc"} 2024-07-03T15:12:18.374Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "526c4321-2a75-46fe-9869-997891b06973", "object": "some-name-pxc"} 2024-07-03T15:12:18.575Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5e6d0744-5c35-4b6c-8a90-e9be8cfdba43", "object": "some-name-pxc"} 2024-07-03T15:12:18.628Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5e6d0744-5c35-4b6c-8a90-e9be8cfdba43", "object": "some-name-pxc"} 2024-07-03T15:12:23.592Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5b3b240c-aa93-41c4-8d95-7bac517daa67", "object": "some-name-pxc"} 2024-07-03T15:12:23.655Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5b3b240c-aa93-41c4-8d95-7bac517daa67", "object": "some-name-pxc"} 2024-07-03T15:12:47.350Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "31b572f1-4282-4b8c-8288-edf19c512b26", "object": "some-name-pxc"} 2024-07-03T15:12:47.422Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "31b572f1-4282-4b8c-8288-edf19c512b26", "object": "some-name-pxc"} 2024-07-03T15:12:52.354Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9be5162f-667c-46fc-b426-42132909dc58", "object": "some-name-pxc"} 2024-07-03T15:12:52.410Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9be5162f-667c-46fc-b426-42132909dc58", "object": "some-name-pxc"} 2024-07-03T15:12:53.157Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "31b572f1-4282-4b8c-8288-edf19c512b26"} 2024-07-03T15:12:57.614Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "82ce1d72-a558-46ea-bd04-afe26acea0eb", "object": "some-name-pxc"} 2024-07-03T15:12:57.683Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "82ce1d72-a558-46ea-bd04-afe26acea0eb", "object": "some-name-pxc"} 2024-07-03T15:13:01.288Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "82ce1d72-a558-46ea-bd04-afe26acea0eb"} 2024-07-03T15:13:02.883Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4ceb0635-2f0b-44b2-a9ca-e17791380d60", "object": "some-name-pxc"} 2024-07-03T15:13:02.948Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4ceb0635-2f0b-44b2-a9ca-e17791380d60", "object": "some-name-pxc"} 2024-07-03T15:13:06.577Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4ceb0635-2f0b-44b2-a9ca-e17791380d60"} 2024-07-03T15:13:08.140Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "08df7d93-4136-44f2-81ec-78981a05e166", "object": "some-name-pxc"} 2024-07-03T15:13:08.206Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "08df7d93-4136-44f2-81ec-78981a05e166", "object": "some-name-pxc"} 2024-07-03T15:13:09.765Z INFO Created user secrets {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "secrets": "my-cluster-secrets-2"} 2024-07-03T15:13:09.765Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "root"} 2024-07-03T15:13:09.797Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "root"} 2024-07-03T15:13:09.816Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "secret": "some-name-mysql-init", "user": "root"} 2024-07-03T15:13:12.772Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "08df7d93-4136-44f2-81ec-78981a05e166", "error": "exec syncusers: command terminated with exit code 1 / \nSyncing user accounts from PXC(some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local:3306) to ProxySQL\nRemoving existing user from ProxySQL: root\nAdding user to ProxySQL: root\n / ERROR 1045 (28000) at line 1: ProxySQL Admin Error: UNIQUE constraint failed: mysql_users.username, mysql_users.frontend\nERROR (line:715) : Failed to add the user (root) from PXC to ProxySQL database. \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / \nSyncing user accounts from PXC(some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local:3306) to ProxySQL\nRemoving existing user from ProxySQL: root\nAdding user to ProxySQL: root\n / ERROR 1045 (28000) at line 1: ProxySQL Admin Error: UNIQUE constraint failed: mysql_users.username, mysql_users.frontend\nERROR (line:715) : Failed to add the user (root) from PXC to ProxySQL database. \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-03T15:13:15.206Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048"} 2024-07-03T15:13:15.219Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "root"} 2024-07-03T15:13:15.219Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "operator"} 2024-07-03T15:13:15.245Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "operator"} 2024-07-03T15:13:15.254Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "secret": "some-name-mysql-init", "user": "operator"} 2024-07-03T15:13:15.263Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "operator"} 2024-07-03T15:13:15.263Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "monitor"} 2024-07-03T15:13:15.287Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "monitor"} 2024-07-03T15:13:15.297Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "secret": "some-name-mysql-init", "user": "monitor"} 2024-07-03T15:13:15.348Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "monitor"} 2024-07-03T15:13:15.358Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "monitor"} 2024-07-03T15:13:15.358Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "xtrabackup"} 2024-07-03T15:13:15.390Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "xtrabackup"} 2024-07-03T15:13:15.398Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "secret": "some-name-mysql-init", "user": "xtrabackup"} 2024-07-03T15:13:15.413Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "xtrabackup"} 2024-07-03T15:13:15.413Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "replication"} 2024-07-03T15:13:15.437Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "replication"} 2024-07-03T15:13:15.448Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "secret": "some-name-mysql-init", "user": "replication"} 2024-07-03T15:13:15.460Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "replication"} 2024-07-03T15:13:15.460Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "proxyadmin"} 2024-07-03T15:13:15.517Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "proxyadmin"} 2024-07-03T15:13:15.535Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "user": "proxyadmin"} 2024-07-03T15:13:15.535Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "last-applied-secret": "b1c2e4d011b9aa3b1064442bc155bbd60d1f3db3b67f16ca722d4b8823f09933"} 2024-07-03T15:13:15.535Z INFO PXC pods will be restarted {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "last-applied-secret": "b1c2e4d011b9aa3b1064442bc155bbd60d1f3db3b67f16ca722d4b8823f09933"} 2024-07-03T15:13:15.537Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "object": "some-name-pxc"} 2024-07-03T15:13:15.678Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "object": "some-name-pxc"} 2024-07-03T15:13:15.722Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "object": "some-name-proxysql"} 2024-07-03T15:13:15.940Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9f216a86-eb24-488b-b2ba-742ca9384048", "error": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-03T15:13:16.219Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7ce83b04-9605-4e40-83c0-f110cec8c2f9", "object": "some-name-pxc"} 2024-07-03T15:13:16.269Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7ce83b04-9605-4e40-83c0-f110cec8c2f9", "object": "some-name-pxc"} 2024-07-03T15:13:16.745Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "586daeae-02fb-4d7a-9f92-060d2375693f", "object": "some-name-pxc"} 2024-07-03T15:13:16.949Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "586daeae-02fb-4d7a-9f92-060d2375693f", "object": "some-name-pxc"} 2024-07-03T15:13:18.096Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e06e5f67-b17d-4af2-b930-b37301a16302", "object": "some-name-pxc"} 2024-07-03T15:13:18.176Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e06e5f67-b17d-4af2-b930-b37301a16302", "object": "some-name-pxc"} 2024-07-03T15:13:21.221Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9e6629af-29d8-4e69-be15-b52c6aafb55e", "object": "some-name-pxc"} 2024-07-03T15:13:21.292Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9e6629af-29d8-4e69-be15-b52c6aafb55e", "object": "some-name-pxc"} 2024-07-03T15:13:26.907Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8e7c0ad9-aea9-41b3-b6d5-97c16485d04f", "object": "some-name-pxc"} 2024-07-03T15:13:27.048Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8e7c0ad9-aea9-41b3-b6d5-97c16485d04f", "object": "some-name-pxc"} 2024-07-03T15:13:32.206Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8c6f4d4c-4ec4-47c0-ae70-86a005e9d893", "object": "some-name-pxc"} 2024-07-03T15:13:32.263Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8c6f4d4c-4ec4-47c0-ae70-86a005e9d893", "object": "some-name-pxc"} 2024-07-03T15:13:32.396Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "85422d4f-c71e-459b-a139-6c2fde1503de", "object": "some-name-pxc"} 2024-07-03T15:13:32.449Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "85422d4f-c71e-459b-a139-6c2fde1503de", "object": "some-name-pxc"} 2024-07-03T15:13:37.403Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "eeadb669-1224-4010-8409-db4dd9759b7c", "object": "some-name-pxc"} 2024-07-03T15:13:37.470Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "eeadb669-1224-4010-8409-db4dd9759b7c", "object": "some-name-pxc"} 2024-07-03T15:13:42.621Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3a4396a5-e1a3-485e-acb9-52e0b7100081", "object": "some-name-pxc"} 2024-07-03T15:13:42.693Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3a4396a5-e1a3-485e-acb9-52e0b7100081", "object": "some-name-pxc"} 2024-07-03T15:13:47.886Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "69a34713-ca1b-403c-b66d-8afa0ae3fab3", "object": "some-name-pxc"} 2024-07-03T15:13:47.972Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "69a34713-ca1b-403c-b66d-8afa0ae3fab3", "object": "some-name-pxc"} 2024-07-03T15:13:53.111Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d0188c5d-f135-4cc0-a550-59f811038390", "object": "some-name-pxc"} 2024-07-03T15:13:53.177Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d0188c5d-f135-4cc0-a550-59f811038390", "object": "some-name-pxc"} 2024-07-03T15:13:58.401Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9814080d-0621-4f54-a42b-4f355e15206e", "object": "some-name-pxc"} 2024-07-03T15:13:58.500Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9814080d-0621-4f54-a42b-4f355e15206e", "object": "some-name-pxc"} 2024-07-03T15:14:03.743Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ca330b22-2829-4e58-a398-517b76d58473", "object": "some-name-pxc"} 2024-07-03T15:14:03.805Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ca330b22-2829-4e58-a398-517b76d58473", "object": "some-name-pxc"} 2024-07-03T15:14:04.189Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "25249702-dccc-4d4c-8ee1-735d4a66e332", "object": "some-name-pxc"} 2024-07-03T15:14:04.254Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "25249702-dccc-4d4c-8ee1-735d4a66e332", "object": "some-name-pxc"} 2024-07-03T15:14:09.190Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d393f855-cf21-4a08-990d-a723213ef275", "object": "some-name-pxc"} 2024-07-03T15:14:09.255Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d393f855-cf21-4a08-990d-a723213ef275", "object": "some-name-pxc"} 2024-07-03T15:14:14.452Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "022bd73e-902f-4a4d-8eca-2397c0d11b13", "object": "some-name-pxc"} 2024-07-03T15:14:14.514Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "022bd73e-902f-4a4d-8eca-2397c0d11b13", "object": "some-name-pxc"} 2024-07-03T15:14:19.688Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e84edd6a-8d56-402a-bc9a-c361472c3e0c", "object": "some-name-pxc"} 2024-07-03T15:14:19.760Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e84edd6a-8d56-402a-bc9a-c361472c3e0c", "object": "some-name-pxc"} 2024-07-03T15:14:25.031Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "da47e5fb-d510-4914-aea8-7ad33b78b312", "object": "some-name-pxc"} 2024-07-03T15:14:25.172Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "da47e5fb-d510-4914-aea8-7ad33b78b312", "object": "some-name-pxc"} 2024-07-03T15:14:30.390Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f24cb698-b444-40a2-bbdc-0802493fba87", "object": "some-name-pxc"} 2024-07-03T15:14:30.450Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f24cb698-b444-40a2-bbdc-0802493fba87", "object": "some-name-pxc"} 2024-07-03T15:14:35.619Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3af99fd9-c5fd-4447-a285-ed5b4eab42bb", "object": "some-name-pxc"} 2024-07-03T15:14:35.802Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3af99fd9-c5fd-4447-a285-ed5b4eab42bb", "object": "some-name-pxc"} 2024-07-03T15:14:41.268Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5de39887-4dcb-4445-8a0e-1e1418fc9515", "object": "some-name-pxc"} 2024-07-03T15:14:41.423Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5de39887-4dcb-4445-8a0e-1e1418fc9515", "object": "some-name-pxc"} 2024-07-03T15:14:46.613Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7ee3b920-f571-4e73-95cd-8e80ec4e19c9", "object": "some-name-pxc"} 2024-07-03T15:14:46.677Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7ee3b920-f571-4e73-95cd-8e80ec4e19c9", "object": "some-name-pxc"} 2024-07-03T15:14:51.841Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e6c8ed73-9863-4100-9884-39d35a250a34", "object": "some-name-pxc"} 2024-07-03T15:14:51.906Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e6c8ed73-9863-4100-9884-39d35a250a34", "object": "some-name-pxc"} 2024-07-03T15:14:51.998Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e6c8ed73-9863-4100-9884-39d35a250a34", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-19584 on 10.69.192.10:53: no such host"} 2024-07-03T15:14:52.190Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "49390914-608b-4a69-9c9d-79de5bfb918b", "object": "some-name-pxc"} 2024-07-03T15:14:52.249Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "49390914-608b-4a69-9c9d-79de5bfb918b", "object": "some-name-pxc"} 2024-07-03T15:14:52.369Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "49390914-608b-4a69-9c9d-79de5bfb918b", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-19584 on 10.69.192.10:53: no such host"} 2024-07-03T15:14:57.143Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ffed7e01-3ccf-490f-98cf-82ab056ea56d", "object": "some-name-pxc"} 2024-07-03T15:14:57.198Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ffed7e01-3ccf-490f-98cf-82ab056ea56d", "object": "some-name-pxc"} 2024-07-03T15:14:57.328Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ffed7e01-3ccf-490f-98cf-82ab056ea56d", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-19584 on 10.69.192.10:53: no such host"} 2024-07-03T15:15:02.445Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "740d8a41-105f-40b3-bbc0-0cd7a3b247ec", "object": "some-name-pxc"} 2024-07-03T15:15:02.500Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "740d8a41-105f-40b3-bbc0-0cd7a3b247ec", "object": "some-name-pxc"} 2024-07-03T15:15:02.580Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "740d8a41-105f-40b3-bbc0-0cd7a3b247ec", "err": "failed to connect to pod some-name-pxc-0: dial tcp: lookup some-name-pxc-0.some-name-pxc.users-19584 on 10.69.192.10:53: no such host"} 2024-07-03T15:15:07.676Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a7250829-dcbb-4a3c-9152-caf48eec7bef", "object": "some-name-pxc"} 2024-07-03T15:15:07.728Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a7250829-dcbb-4a3c-9152-caf48eec7bef", "object": "some-name-pxc"} 2024-07-03T15:15:07.784Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a7250829-dcbb-4a3c-9152-caf48eec7bef", "primary name": "some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local"} 2024-07-03T15:15:12.857Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "98a193ca-c473-4635-8a36-d17efb56b703", "object": "some-name-pxc"} 2024-07-03T15:15:12.923Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "98a193ca-c473-4635-8a36-d17efb56b703", "object": "some-name-pxc"} 2024-07-03T15:15:12.969Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "98a193ca-c473-4635-8a36-d17efb56b703", "primary name": "some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local"} 2024-07-03T15:15:18.040Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "09c5660a-c4b9-4b06-9316-d75e5034ebf9", "object": "some-name-pxc"} 2024-07-03T15:15:18.107Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "09c5660a-c4b9-4b06-9316-d75e5034ebf9", "object": "some-name-pxc"} 2024-07-03T15:15:18.163Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "09c5660a-c4b9-4b06-9316-d75e5034ebf9", "primary name": "some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local"} 2024-07-03T15:15:23.239Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e77b6702-7417-41d2-8bdb-4443b7642a81", "object": "some-name-pxc"} 2024-07-03T15:15:23.293Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e77b6702-7417-41d2-8bdb-4443b7642a81", "object": "some-name-pxc"} 2024-07-03T15:15:23.391Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e77b6702-7417-41d2-8bdb-4443b7642a81", "primary name": "some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local"} 2024-07-03T15:15:28.571Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "eea2eeb5-41f9-4002-8791-6dc6b148f58f", "object": "some-name-pxc"} 2024-07-03T15:15:28.634Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "eea2eeb5-41f9-4002-8791-6dc6b148f58f", "object": "some-name-pxc"} 2024-07-03T15:15:28.685Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "eea2eeb5-41f9-4002-8791-6dc6b148f58f", "primary name": "some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local"} 2024-07-03T15:15:33.764Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6ce915b2-4144-486b-af0f-c199c139c4e4", "object": "some-name-pxc"} 2024-07-03T15:15:33.824Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6ce915b2-4144-486b-af0f-c199c139c4e4", "object": "some-name-pxc"} 2024-07-03T15:15:33.874Z INFO Unable to find primary pod for replication. No pod with name or ip like this {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6ce915b2-4144-486b-af0f-c199c139c4e4", "primary name": "some-name-pxc-0.some-name-pxc.users-19584.svc.cluster.local"} 2024-07-03T15:15:38.992Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fc5b634b-1529-460e-ad60-569337d228d1", "object": "some-name-pxc"} 2024-07-03T15:15:39.049Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fc5b634b-1529-460e-ad60-569337d228d1", "object": "some-name-pxc"} 2024-07-03T15:15:39.334Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ab10a7f2-4c65-4ed6-9d7b-1d1dab246a73", "object": "some-name-pxc"} 2024-07-03T15:15:39.398Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ab10a7f2-4c65-4ed6-9d7b-1d1dab246a73", "object": "some-name-pxc"} 2024-07-03T15:15:43.080Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ab10a7f2-4c65-4ed6-9d7b-1d1dab246a73"} 2024-07-03T15:15:44.334Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "851083e9-ee08-4387-90b8-839c1a02f7c7", "object": "some-name-pxc"} 2024-07-03T15:15:44.410Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "851083e9-ee08-4387-90b8-839c1a02f7c7", "object": "some-name-pxc"} 2024-07-03T15:15:47.990Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "851083e9-ee08-4387-90b8-839c1a02f7c7"} 2024-07-03T15:15:49.660Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "04c56237-c117-47f5-a226-8731e65a0ba0", "object": "some-name-pxc"} 2024-07-03T15:15:49.736Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "04c56237-c117-47f5-a226-8731e65a0ba0", "object": "some-name-pxc"} 2024-07-03T15:15:53.355Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "04c56237-c117-47f5-a226-8731e65a0ba0"} 2024-07-03T15:15:54.923Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f426fc58-420c-4da8-ab4f-b2e51ba9fcef", "object": "some-name-pxc"} 2024-07-03T15:15:55.005Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f426fc58-420c-4da8-ab4f-b2e51ba9fcef", "object": "some-name-pxc"} 2024-07-03T15:15:58.756Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f426fc58-420c-4da8-ab4f-b2e51ba9fcef"} 2024-07-03T15:16:00.192Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07bec7b5-ca78-40bd-919c-8ddf4c211d55", "user": "operator"} 2024-07-03T15:16:00.217Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07bec7b5-ca78-40bd-919c-8ddf4c211d55", "user": "operator"} 2024-07-03T15:16:00.226Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07bec7b5-ca78-40bd-919c-8ddf4c211d55", "secret": "some-name-mysql-init", "user": "operator"} 2024-07-03T15:16:00.234Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07bec7b5-ca78-40bd-919c-8ddf4c211d55", "user": "operator"} 2024-07-03T15:16:00.235Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07bec7b5-ca78-40bd-919c-8ddf4c211d55", "last-applied-secret": "746b5e1b9a907bf8c8b6bbded7872b4088c9ea087b835f4375acfa6c7b538ec3"} 2024-07-03T15:16:00.237Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07bec7b5-ca78-40bd-919c-8ddf4c211d55", "object": "some-name-pxc"} 2024-07-03T15:16:00.306Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07bec7b5-ca78-40bd-919c-8ddf4c211d55", "object": "some-name-pxc"} 2024-07-03T15:16:00.344Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07bec7b5-ca78-40bd-919c-8ddf4c211d55", "object": "some-name-proxysql"} 2024-07-03T15:16:00.678Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "423e4ded-6b78-494e-bb46-0d7b4904d608", "object": "some-name-pxc"} 2024-07-03T15:16:00.744Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "423e4ded-6b78-494e-bb46-0d7b4904d608", "object": "some-name-pxc"} 2024-07-03T15:16:01.885Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07bec7b5-ca78-40bd-919c-8ddf4c211d55", "error": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-19584.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-19584.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-19584.svc.' (using password: YES)\nERROR (line:569) : Could not find a primary cluster node\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-19584.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-19584.svc.' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.users-19584.svc.' (using password: YES)\nERROR (line:569) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-03T15:16:05.671Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a8bf69cd-c038-4790-8c9d-23f1da6aa78b", "object": "some-name-pxc"} 2024-07-03T15:16:05.727Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a8bf69cd-c038-4790-8c9d-23f1da6aa78b", "object": "some-name-pxc"} 2024-07-03T15:16:06.198Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "764a4102-5102-4ed3-862d-077ada08d565", "object": "some-name-pxc"} 2024-07-03T15:16:06.389Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "764a4102-5102-4ed3-862d-077ada08d565", "object": "some-name-pxc"} 2024-07-03T15:16:38.676Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2b26e6c5-1487-4725-8339-3d5031e0bd6c", "object": "some-name-pxc"} 2024-07-03T15:16:38.774Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2b26e6c5-1487-4725-8339-3d5031e0bd6c", "object": "some-name-pxc"} 2024-07-03T15:16:43.649Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "12dd1262-514e-4956-b520-25f493c402cd", "object": "some-name-pxc"} 2024-07-03T15:16:43.726Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "12dd1262-514e-4956-b520-25f493c402cd", "object": "some-name-pxc"} 2024-07-03T15:16:44.749Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2b26e6c5-1487-4725-8339-3d5031e0bd6c"} 2024-07-03T15:16:48.966Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ac7ffed5-c76b-430e-a2b0-729ec222f404", "object": "some-name-pxc"} 2024-07-03T15:16:49.039Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ac7ffed5-c76b-430e-a2b0-729ec222f404", "object": "some-name-pxc"} 2024-07-03T15:16:52.808Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ac7ffed5-c76b-430e-a2b0-729ec222f404"} 2024-07-03T15:16:54.229Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "31d111ab-a362-4239-8c2e-1af53e268e3a", "object": "some-name-pxc"} 2024-07-03T15:16:54.292Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "31d111ab-a362-4239-8c2e-1af53e268e3a", "object": "some-name-pxc"} 2024-07-03T15:16:58.856Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "31d111ab-a362-4239-8c2e-1af53e268e3a"} 2024-07-03T15:16:59.474Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "931a8850-e584-4a67-b13f-c957c100e57f", "object": "some-name-pxc"} 2024-07-03T15:16:59.540Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "931a8850-e584-4a67-b13f-c957c100e57f", "object": "some-name-pxc"} 2024-07-03T15:17:03.167Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "931a8850-e584-4a67-b13f-c957c100e57f"} 2024-07-03T15:17:04.747Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "11646d64-ef68-43a7-9a7b-48e3b8e45172", "object": "some-name-pxc"} 2024-07-03T15:17:04.828Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "11646d64-ef68-43a7-9a7b-48e3b8e45172", "object": "some-name-pxc"} 2024-07-03T15:17:08.375Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "11646d64-ef68-43a7-9a7b-48e3b8e45172"} 2024-07-03T15:17:10.007Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b30aa805-02f6-4335-ad96-2b55682cc5e8", "object": "some-name-pxc"} 2024-07-03T15:17:10.084Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b30aa805-02f6-4335-ad96-2b55682cc5e8", "object": "some-name-pxc"} 2024-07-03T15:17:15.324Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6ceb98bd-0b8a-446d-9309-7d8a987f9e7f", "object": "some-name-pxc"} 2024-07-03T15:17:15.383Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6ceb98bd-0b8a-446d-9309-7d8a987f9e7f", "object": "some-name-pxc"} 2024-07-03T15:17:15.656Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b30aa805-02f6-4335-ad96-2b55682cc5e8"} 2024-07-03T15:17:20.602Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b975156e-645b-41d2-b4bf-936f71e1c895", "object": "some-name-pxc"} 2024-07-03T15:17:20.707Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b975156e-645b-41d2-b4bf-936f71e1c895", "object": "some-name-pxc"} 2024-07-03T15:17:24.292Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b975156e-645b-41d2-b4bf-936f71e1c895"} 2024-07-03T15:17:25.929Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e52b1350-4ae6-4e9b-91e2-c62341ebf351", "object": "some-name-pxc"} 2024-07-03T15:17:26.756Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e52b1350-4ae6-4e9b-91e2-c62341ebf351", "object": "some-name-pxc"} 2024-07-03T15:17:30.365Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e52b1350-4ae6-4e9b-91e2-c62341ebf351"} 2024-07-03T15:17:32.126Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "16217374-e571-4b8c-a5eb-e839b3e879a4", "object": "some-name-pxc"} 2024-07-03T15:17:32.198Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "16217374-e571-4b8c-a5eb-e839b3e879a4", "object": "some-name-pxc"} 2024-07-03T15:17:35.671Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "16217374-e571-4b8c-a5eb-e839b3e879a4"} 2024-07-03T15:17:37.405Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "da8ac891-ecea-4719-a710-737dac335045", "object": "some-name-pxc"} 2024-07-03T15:17:37.484Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "da8ac891-ecea-4719-a710-737dac335045", "object": "some-name-pxc"} 2024-07-03T15:17:41.110Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "da8ac891-ecea-4719-a710-737dac335045"} 2024-07-03T15:17:42.718Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5f197342-7b11-443c-b994-e2c9e9f229d8", "object": "some-name-pxc"} 2024-07-03T15:17:42.830Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5f197342-7b11-443c-b994-e2c9e9f229d8", "object": "some-name-pxc"} 2024-07-03T15:17:46.400Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5f197342-7b11-443c-b994-e2c9e9f229d8"} 2024-07-03T15:17:48.104Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b59e03f6-4790-4e40-b94d-37b91460e74e", "object": "some-name-pxc"} 2024-07-03T15:17:48.190Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b59e03f6-4790-4e40-b94d-37b91460e74e", "object": "some-name-pxc"} 2024-07-03T15:17:51.697Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b59e03f6-4790-4e40-b94d-37b91460e74e"} 2024-07-03T15:17:53.390Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f37c6533-2601-44d1-9637-d2102998dbb0", "object": "some-name-pxc"} 2024-07-03T15:17:53.479Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f37c6533-2601-44d1-9637-d2102998dbb0", "object": "some-name-pxc"} 2024-07-03T15:17:56.975Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f37c6533-2601-44d1-9637-d2102998dbb0"} 2024-07-03T15:17:58.744Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "root"} 2024-07-03T15:17:58.776Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "root"} 2024-07-03T15:17:58.789Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "secret": "some-name-mysql-init", "user": "root"} 2024-07-03T15:18:04.249Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496"} 2024-07-03T15:18:04.263Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "root"} 2024-07-03T15:18:04.263Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "monitor"} 2024-07-03T15:18:04.288Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "monitor"} 2024-07-03T15:18:04.306Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "secret": "some-name-mysql-init", "user": "monitor"} 2024-07-03T15:18:04.357Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "monitor"} 2024-07-03T15:18:04.378Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "monitor"} 2024-07-03T15:18:04.378Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "xtrabackup"} 2024-07-03T15:18:04.399Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "xtrabackup"} 2024-07-03T15:18:04.442Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "secret": "some-name-mysql-init", "user": "xtrabackup"} 2024-07-03T15:18:04.458Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "xtrabackup"} 2024-07-03T15:18:04.458Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "proxyadmin"} 2024-07-03T15:18:04.517Z INFO Proxy user updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "proxyadmin"} 2024-07-03T15:18:04.529Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "user": "proxyadmin"} 2024-07-03T15:18:04.529Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "last-applied-secret": "8222bc1a41d6a14f0d6bfdb7715c0f1a9a95fb51374cf9ab91d137a928891c92"} 2024-07-03T15:18:04.529Z INFO PXC pods will be restarted {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "last-applied-secret": "8222bc1a41d6a14f0d6bfdb7715c0f1a9a95fb51374cf9ab91d137a928891c92"} 2024-07-03T15:18:04.533Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "object": "some-name-pxc"} 2024-07-03T15:18:04.619Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "object": "some-name-pxc"} 2024-07-03T15:18:04.675Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "object": "some-name-proxysql"} 2024-07-03T15:18:04.904Z ERROR sync users {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "296c98b5-af9b-438c-95e7-bd2e7b115496", "error": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n", "errorVerbose": "exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): ProxySQL Error: Access denied for user 'proxyadmin'@'127.0.0.1' (using password: YES)\nERROR (line:335) : ProxySQL connection check failed. \n-- Could not connect to ProxySQL at localhost:6032 \n-- Please check the ProxySQL connection parameters and status.\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-07-03T15:18:05.884Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7d377ca0-d990-4bf3-b126-1e55d7901161", "object": "some-name-pxc"} 2024-07-03T15:18:06.042Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7d377ca0-d990-4bf3-b126-1e55d7901161", "object": "some-name-pxc"} 2024-07-03T15:18:06.490Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ac2cf202-d104-4b46-93a7-8b4865babafd", "object": "some-name-pxc"} 2024-07-03T15:18:06.584Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ac2cf202-d104-4b46-93a7-8b4865babafd", "object": "some-name-pxc"} 2024-07-03T15:18:10.734Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a8b8588e-c6a9-4e24-9fbb-0b2b51085977", "object": "some-name-pxc"} 2024-07-03T15:18:10.818Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a8b8588e-c6a9-4e24-9fbb-0b2b51085977", "object": "some-name-pxc"} 2024-07-03T15:18:16.681Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "771dfc68-0e25-4420-a394-1af1f447addb", "object": "some-name-pxc"} 2024-07-03T15:18:16.775Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "771dfc68-0e25-4420-a394-1af1f447addb", "object": "some-name-pxc"} 2024-07-03T15:18:18.593Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "624f033c-9871-4ecf-8ddc-c3e433970bd9", "object": "some-name-pxc"} 2024-07-03T15:18:18.679Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "624f033c-9871-4ecf-8ddc-c3e433970bd9", "object": "some-name-haproxy"} 2024-07-03T15:18:18.696Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "624f033c-9871-4ecf-8ddc-c3e433970bd9", "object": "some-name-pxc"} 2024-07-03T15:18:18.975Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c1cd08ae-286d-4a59-a58b-f005049ac7f2", "object": "some-name-pxc"} 2024-07-03T15:18:19.045Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c1cd08ae-286d-4a59-a58b-f005049ac7f2", "object": "some-name-haproxy"} 2024-07-03T15:18:19.107Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c1cd08ae-286d-4a59-a58b-f005049ac7f2", "object": "some-name-haproxy"} 2024-07-03T15:18:19.248Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c1cd08ae-286d-4a59-a58b-f005049ac7f2", "object": "some-name-haproxy-replicas"} 2024-07-03T15:18:19.248Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c1cd08ae-286d-4a59-a58b-f005049ac7f2", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:18:19.481Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: ed88df52-dbee-440a-88bf-7b11eadf9db0 2024-07-03T15:18:20.696Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "41cd6c64-cf88-4529-ac4c-2c226a338fba", "object": "some-name-pxc"} 2024-07-03T15:18:20.940Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6ff12d5c-feb8-4276-a09f-e1e14ddac77e", "object": "some-name-pxc"} 2024-07-03T15:18:21.037Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6ff12d5c-feb8-4276-a09f-e1e14ddac77e", "object": "some-name-pxc"} 2024-07-03T15:18:21.126Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6ff12d5c-feb8-4276-a09f-e1e14ddac77e", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:18:21.277Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e4fb40d9-4466-422a-88aa-2e9c6c5e2be9", "object": "some-name-pxc"} 2024-07-03T15:18:21.343Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e4fb40d9-4466-422a-88aa-2e9c6c5e2be9", "object": "some-name-pxc"} 2024-07-03T15:18:21.383Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e4fb40d9-4466-422a-88aa-2e9c6c5e2be9", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:18:26.795Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "241817ab-9801-4d7c-ab95-623855a75319", "object": "some-name-pxc"} 2024-07-03T15:18:27.246Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "241817ab-9801-4d7c-ab95-623855a75319", "object": "some-name-pxc"} 2024-07-03T15:18:27.393Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "241817ab-9801-4d7c-ab95-623855a75319", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:18:32.600Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "805357b6-b3fb-40c8-9219-dd6442a09e3a", "object": "some-name-pxc"} 2024-07-03T15:18:32.681Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "805357b6-b3fb-40c8-9219-dd6442a09e3a", "object": "some-name-pxc"} 2024-07-03T15:18:32.722Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "805357b6-b3fb-40c8-9219-dd6442a09e3a", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:18:37.845Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "91e8779e-2c15-4013-87d8-bd1f13b59b47", "object": "some-name-pxc"} 2024-07-03T15:18:37.914Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "91e8779e-2c15-4013-87d8-bd1f13b59b47", "object": "some-name-pxc"} 2024-07-03T15:18:37.951Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "91e8779e-2c15-4013-87d8-bd1f13b59b47", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:18:43.063Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "558acb13-ce49-4c7f-871b-f74678754151", "object": "some-name-pxc"} 2024-07-03T15:18:43.160Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "558acb13-ce49-4c7f-871b-f74678754151", "object": "some-name-pxc"} 2024-07-03T15:18:43.212Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "558acb13-ce49-4c7f-871b-f74678754151", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:18:43.571Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60440044-540a-4d0f-8281-85fa3f279284", "object": "some-name-pxc"} 2024-07-03T15:18:43.770Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60440044-540a-4d0f-8281-85fa3f279284", "object": "some-name-pxc"} 2024-07-03T15:18:43.846Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60440044-540a-4d0f-8281-85fa3f279284", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:18:48.460Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4ed9e4fc-2c16-425c-a22e-84b1b78c1788", "object": "some-name-pxc"} 2024-07-03T15:18:48.525Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4ed9e4fc-2c16-425c-a22e-84b1b78c1788", "object": "some-name-pxc"} 2024-07-03T15:18:48.562Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4ed9e4fc-2c16-425c-a22e-84b1b78c1788", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:18:53.685Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2a8ba458-ff19-4aab-a113-7022b27d3432", "object": "some-name-pxc"} 2024-07-03T15:18:53.757Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2a8ba458-ff19-4aab-a113-7022b27d3432", "object": "some-name-pxc"} 2024-07-03T15:18:53.785Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2a8ba458-ff19-4aab-a113-7022b27d3432", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:18:58.927Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8bb64b66-4395-409c-a1db-bd3767b0db64", "object": "some-name-pxc"} 2024-07-03T15:18:58.994Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8bb64b66-4395-409c-a1db-bd3767b0db64", "object": "some-name-pxc"} 2024-07-03T15:18:59.027Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8bb64b66-4395-409c-a1db-bd3767b0db64", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:04.209Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1906faaa-82c7-43dc-9299-fceec1187937", "object": "some-name-pxc"} 2024-07-03T15:19:04.293Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1906faaa-82c7-43dc-9299-fceec1187937", "object": "some-name-pxc"} 2024-07-03T15:19:04.333Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1906faaa-82c7-43dc-9299-fceec1187937", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:04.647Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "086834a3-5ab3-4426-b0c1-a1d6d6d3baec", "object": "some-name-pxc"} 2024-07-03T15:19:04.736Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "086834a3-5ab3-4426-b0c1-a1d6d6d3baec", "object": "some-name-pxc"} 2024-07-03T15:19:04.780Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "086834a3-5ab3-4426-b0c1-a1d6d6d3baec", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:09.665Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "43f21e50-e060-4d4b-be4b-1f816ecd2051", "object": "some-name-pxc"} 2024-07-03T15:19:09.722Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "43f21e50-e060-4d4b-be4b-1f816ecd2051", "object": "some-name-pxc"} 2024-07-03T15:19:09.775Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "43f21e50-e060-4d4b-be4b-1f816ecd2051", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:14.906Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "0a2b199c-d739-4e34-bd75-03cc1fdcc680", "object": "some-name-pxc"} 2024-07-03T15:19:14.997Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "0a2b199c-d739-4e34-bd75-03cc1fdcc680", "object": "some-name-pxc"} 2024-07-03T15:19:15.038Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "0a2b199c-d739-4e34-bd75-03cc1fdcc680", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:20.165Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f7a34b37-1609-4818-819f-68d4240d8e34", "object": "some-name-pxc"} 2024-07-03T15:19:20.213Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f7a34b37-1609-4818-819f-68d4240d8e34", "object": "some-name-pxc"} 2024-07-03T15:19:20.246Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f7a34b37-1609-4818-819f-68d4240d8e34", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:25.409Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ba0c4749-119b-4be6-b266-6404604a9b6a", "object": "some-name-pxc"} 2024-07-03T15:19:25.499Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ba0c4749-119b-4be6-b266-6404604a9b6a", "object": "some-name-pxc"} 2024-07-03T15:19:25.546Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ba0c4749-119b-4be6-b266-6404604a9b6a", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:25.732Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5f6d00ff-4136-40c9-86fd-825fa7b6b162", "object": "some-name-pxc"} 2024-07-03T15:19:25.790Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5f6d00ff-4136-40c9-86fd-825fa7b6b162", "object": "some-name-pxc"} 2024-07-03T15:19:25.827Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5f6d00ff-4136-40c9-86fd-825fa7b6b162", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:30.776Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "89b750d7-48b4-493d-9b6a-9ab06afcca90", "object": "some-name-pxc"} 2024-07-03T15:19:30.890Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "89b750d7-48b4-493d-9b6a-9ab06afcca90", "object": "some-name-pxc"} 2024-07-03T15:19:30.939Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "89b750d7-48b4-493d-9b6a-9ab06afcca90", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:36.130Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "57b42d56-783f-4166-8b67-aaa45f3278a3", "object": "some-name-pxc"} 2024-07-03T15:19:36.197Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "57b42d56-783f-4166-8b67-aaa45f3278a3", "object": "some-name-pxc"} 2024-07-03T15:19:36.237Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "57b42d56-783f-4166-8b67-aaa45f3278a3", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:41.423Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "610e1f66-5669-4661-a791-8f8c475d32a0", "object": "some-name-pxc"} 2024-07-03T15:19:41.550Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "610e1f66-5669-4661-a791-8f8c475d32a0", "object": "some-name-pxc"} 2024-07-03T15:19:41.631Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "610e1f66-5669-4661-a791-8f8c475d32a0", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:46.798Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6f5c9789-ac64-44ae-b294-da47fed4714e", "object": "some-name-pxc"} 2024-07-03T15:19:46.882Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6f5c9789-ac64-44ae-b294-da47fed4714e", "object": "some-name-pxc"} 2024-07-03T15:19:46.914Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6f5c9789-ac64-44ae-b294-da47fed4714e", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:52.103Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c924497b-3c73-426c-877a-dbd4739e3a44", "object": "some-name-pxc"} 2024-07-03T15:19:52.182Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c924497b-3c73-426c-877a-dbd4739e3a44", "object": "some-name-pxc"} 2024-07-03T15:19:52.216Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c924497b-3c73-426c-877a-dbd4739e3a44", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:57.390Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9b1b49ce-caa8-4f14-8140-25db362437ac", "object": "some-name-pxc"} 2024-07-03T15:19:57.527Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9b1b49ce-caa8-4f14-8140-25db362437ac", "object": "some-name-pxc"} 2024-07-03T15:19:57.590Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9b1b49ce-caa8-4f14-8140-25db362437ac", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:19:57.854Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7a51ac04-649a-4c55-831f-7e79b52392d3", "object": "some-name-pxc"} 2024-07-03T15:19:57.938Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7a51ac04-649a-4c55-831f-7e79b52392d3", "object": "some-name-pxc"} 2024-07-03T15:19:57.977Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7a51ac04-649a-4c55-831f-7e79b52392d3", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:02.832Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ee9ece82-4008-4a61-9921-85387ced6bb2", "object": "some-name-pxc"} 2024-07-03T15:20:02.897Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ee9ece82-4008-4a61-9921-85387ced6bb2", "object": "some-name-pxc"} 2024-07-03T15:20:02.960Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ee9ece82-4008-4a61-9921-85387ced6bb2", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:08.111Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8de04893-aff9-4052-a3ae-6478d676697d", "object": "some-name-pxc"} 2024-07-03T15:20:08.173Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8de04893-aff9-4052-a3ae-6478d676697d", "object": "some-name-pxc"} 2024-07-03T15:20:08.209Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8de04893-aff9-4052-a3ae-6478d676697d", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:13.322Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "af69f44c-9161-46c5-86dc-36531c3be204", "object": "some-name-pxc"} 2024-07-03T15:20:13.397Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "af69f44c-9161-46c5-86dc-36531c3be204", "object": "some-name-pxc"} 2024-07-03T15:20:13.434Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "af69f44c-9161-46c5-86dc-36531c3be204", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:18.554Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c779bae0-aeaf-4e1b-b821-40e2e3902d89", "object": "some-name-pxc"} 2024-07-03T15:20:18.617Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c779bae0-aeaf-4e1b-b821-40e2e3902d89", "object": "some-name-pxc"} 2024-07-03T15:20:18.657Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c779bae0-aeaf-4e1b-b821-40e2e3902d89", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:23.802Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3959b24f-a5cc-4dba-9091-1de99b36b414", "object": "some-name-pxc"} 2024-07-03T15:20:23.888Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3959b24f-a5cc-4dba-9091-1de99b36b414", "object": "some-name-pxc"} 2024-07-03T15:20:23.933Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3959b24f-a5cc-4dba-9091-1de99b36b414", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:29.046Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9a78ec16-7b57-465e-8d08-9393cfc20b5d", "object": "some-name-pxc"} 2024-07-03T15:20:29.108Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9a78ec16-7b57-465e-8d08-9393cfc20b5d", "object": "some-name-pxc"} 2024-07-03T15:20:29.150Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9a78ec16-7b57-465e-8d08-9393cfc20b5d", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:34.258Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "969093a6-682e-4b90-97eb-712f0b3f2389", "object": "some-name-pxc"} 2024-07-03T15:20:34.320Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "969093a6-682e-4b90-97eb-712f0b3f2389", "object": "some-name-pxc"} 2024-07-03T15:20:34.365Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "969093a6-682e-4b90-97eb-712f0b3f2389", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:39.489Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a24094ab-b525-458c-9b46-f148b4442d90", "object": "some-name-pxc"} 2024-07-03T15:20:39.576Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a24094ab-b525-458c-9b46-f148b4442d90", "object": "some-name-pxc"} 2024-07-03T15:20:39.636Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a24094ab-b525-458c-9b46-f148b4442d90", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:44.821Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5eec0646-ec10-46e2-8d50-942a85fe3c0b", "object": "some-name-pxc"} 2024-07-03T15:20:44.910Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5eec0646-ec10-46e2-8d50-942a85fe3c0b", "object": "some-name-pxc"} 2024-07-03T15:20:44.954Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5eec0646-ec10-46e2-8d50-942a85fe3c0b", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:50.182Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6372795a-b9bb-437a-b753-bbc87ea45ed6", "object": "some-name-pxc"} 2024-07-03T15:20:50.276Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6372795a-b9bb-437a-b753-bbc87ea45ed6", "object": "some-name-pxc"} 2024-07-03T15:20:50.321Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6372795a-b9bb-437a-b753-bbc87ea45ed6", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:50.557Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "root"} 2024-07-03T15:20:50.582Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "root"} 2024-07-03T15:20:50.596Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "secret": "some-name-mysql-init", "user": "root"} 2024-07-03T15:20:50.630Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "root"} 2024-07-03T15:20:50.630Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "operator"} 2024-07-03T15:20:50.659Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "operator"} 2024-07-03T15:20:50.676Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "secret": "some-name-mysql-init", "user": "operator"} 2024-07-03T15:20:50.714Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "operator"} 2024-07-03T15:20:50.715Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "monitor"} 2024-07-03T15:20:50.743Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "monitor"} 2024-07-03T15:20:50.760Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "secret": "some-name-mysql-init", "user": "monitor"} 2024-07-03T15:20:50.793Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "monitor"} 2024-07-03T15:20:50.793Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "xtrabackup"} 2024-07-03T15:20:50.824Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "xtrabackup"} 2024-07-03T15:20:50.835Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "secret": "some-name-mysql-init", "user": "xtrabackup"} 2024-07-03T15:20:50.860Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "xtrabackup"} 2024-07-03T15:20:50.860Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "replication"} 2024-07-03T15:20:50.876Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "replication"} 2024-07-03T15:20:50.891Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "secret": "some-name-mysql-init", "user": "replication"} 2024-07-03T15:20:50.919Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "user": "replication"} 2024-07-03T15:20:50.919Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"} 2024-07-03T15:20:50.919Z INFO PXC pods will be restarted {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "last-applied-secret": "6542c6f8a2375ddfcda319bf91c99ca69519730971c577cd07366e356671ee68"} 2024-07-03T15:20:50.922Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "object": "some-name-pxc"} 2024-07-03T15:20:51.029Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "object": "some-name-pxc"} 2024-07-03T15:20:51.076Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "object": "some-name-haproxy"} 2024-07-03T15:20:51.135Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "60b6fb9f-a9df-4d7c-9fc3-20e842e1c23b", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:51.728Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4d8acc23-1c62-46b7-99cb-51d55a59cde4", "object": "some-name-pxc"} 2024-07-03T15:20:51.838Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4d8acc23-1c62-46b7-99cb-51d55a59cde4", "object": "some-name-pxc"} 2024-07-03T15:20:52.002Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4d8acc23-1c62-46b7-99cb-51d55a59cde4", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:53.120Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cfa8ac35-71fe-4a8f-8fb4-e8881a7f4250", "object": "some-name-pxc"} 2024-07-03T15:20:53.177Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cfa8ac35-71fe-4a8f-8fb4-e8881a7f4250", "object": "some-name-pxc"} 2024-07-03T15:20:53.207Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cfa8ac35-71fe-4a8f-8fb4-e8881a7f4250", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:20:55.536Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b547605b-dc2f-41bb-863f-ce9f542edcf6", "object": "some-name-pxc"} 2024-07-03T15:20:55.613Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b547605b-dc2f-41bb-863f-ce9f542edcf6", "object": "some-name-pxc"} 2024-07-03T15:20:55.657Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b547605b-dc2f-41bb-863f-ce9f542edcf6", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:00.921Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b859fc03-799d-49bb-a0de-c142cc56a879", "object": "some-name-pxc"} 2024-07-03T15:21:01.014Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b859fc03-799d-49bb-a0de-c142cc56a879", "object": "some-name-pxc"} 2024-07-03T15:21:01.081Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b859fc03-799d-49bb-a0de-c142cc56a879", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:06.236Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cdf70fa3-b996-44d9-bae6-f3303a6927ce", "object": "some-name-pxc"} 2024-07-03T15:21:06.376Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cdf70fa3-b996-44d9-bae6-f3303a6927ce", "object": "some-name-pxc"} 2024-07-03T15:21:06.457Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cdf70fa3-b996-44d9-bae6-f3303a6927ce", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:06.629Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b6dae3c5-67d4-4d4c-a003-78f7435c4d7f", "object": "some-name-pxc"} 2024-07-03T15:21:06.736Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b6dae3c5-67d4-4d4c-a003-78f7435c4d7f", "object": "some-name-pxc"} 2024-07-03T15:21:06.811Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b6dae3c5-67d4-4d4c-a003-78f7435c4d7f", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:11.821Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "04bbf26b-27ea-4f34-a2c4-9ee585f3bdf8", "object": "some-name-pxc"} 2024-07-03T15:21:12.205Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "04bbf26b-27ea-4f34-a2c4-9ee585f3bdf8", "object": "some-name-pxc"} 2024-07-03T15:21:12.321Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "04bbf26b-27ea-4f34-a2c4-9ee585f3bdf8", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:12.501Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "01acf75f-0a02-4d04-acc8-d2745ab3fa1a", "object": "some-name-pxc"} 2024-07-03T15:21:12.720Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "01acf75f-0a02-4d04-acc8-d2745ab3fa1a", "object": "some-name-pxc"} 2024-07-03T15:21:12.852Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "01acf75f-0a02-4d04-acc8-d2745ab3fa1a", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:13.021Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d659fa78-71e2-4239-84be-a6ee88a15b4e", "object": "some-name-pxc"} 2024-07-03T15:21:13.167Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d659fa78-71e2-4239-84be-a6ee88a15b4e", "object": "some-name-pxc"} 2024-07-03T15:21:13.206Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d659fa78-71e2-4239-84be-a6ee88a15b4e", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:17.495Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ee48091e-0cb1-4137-a8b0-3cfbd94d6cc9", "object": "some-name-pxc"} 2024-07-03T15:21:17.563Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ee48091e-0cb1-4137-a8b0-3cfbd94d6cc9", "object": "some-name-pxc"} 2024-07-03T15:21:17.595Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ee48091e-0cb1-4137-a8b0-3cfbd94d6cc9", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:17.720Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d813a92c-67eb-446c-b67a-362a9ab3ca2b", "object": "some-name-pxc"} 2024-07-03T15:21:17.794Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d813a92c-67eb-446c-b67a-362a9ab3ca2b", "object": "some-name-pxc"} 2024-07-03T15:21:17.850Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d813a92c-67eb-446c-b67a-362a9ab3ca2b", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:22.724Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7c76a0f8-e1e5-452a-8232-dca8ea0e629e", "object": "some-name-pxc"} 2024-07-03T15:21:22.787Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7c76a0f8-e1e5-452a-8232-dca8ea0e629e", "object": "some-name-pxc"} 2024-07-03T15:21:22.828Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7c76a0f8-e1e5-452a-8232-dca8ea0e629e", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:27.988Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "56b994a3-d205-4426-85b9-4bc04e4ef17f", "object": "some-name-pxc"} 2024-07-03T15:21:28.046Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "56b994a3-d205-4426-85b9-4bc04e4ef17f", "object": "some-name-pxc"} 2024-07-03T15:21:28.080Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "56b994a3-d205-4426-85b9-4bc04e4ef17f", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:33.211Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1da7927c-f2b3-4fb4-9c62-f7ac172136b7", "object": "some-name-pxc"} 2024-07-03T15:21:33.294Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1da7927c-f2b3-4fb4-9c62-f7ac172136b7", "object": "some-name-pxc"} 2024-07-03T15:21:33.325Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "1da7927c-f2b3-4fb4-9c62-f7ac172136b7", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:38.558Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "815df7b4-55d1-4fb6-b702-3d201d31a4d9", "object": "some-name-pxc"} 2024-07-03T15:21:38.654Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "815df7b4-55d1-4fb6-b702-3d201d31a4d9", "object": "some-name-pxc"} 2024-07-03T15:21:38.694Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "815df7b4-55d1-4fb6-b702-3d201d31a4d9", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:43.937Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8bddc9da-1e05-4c32-a657-c110f448b1f5", "object": "some-name-pxc"} 2024-07-03T15:21:44.040Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8bddc9da-1e05-4c32-a657-c110f448b1f5", "object": "some-name-pxc"} 2024-07-03T15:21:44.082Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8bddc9da-1e05-4c32-a657-c110f448b1f5", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:44.290Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c596534f-52ff-4862-bd16-683a24cbaca5", "object": "some-name-pxc"} 2024-07-03T15:21:44.408Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c596534f-52ff-4862-bd16-683a24cbaca5", "object": "some-name-pxc"} 2024-07-03T15:21:44.475Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c596534f-52ff-4862-bd16-683a24cbaca5", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:49.271Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5098e2d4-5d29-4e25-9d4a-7c3193bed044", "object": "some-name-pxc"} 2024-07-03T15:21:49.320Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5098e2d4-5d29-4e25-9d4a-7c3193bed044", "object": "some-name-pxc"} 2024-07-03T15:21:49.360Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5098e2d4-5d29-4e25-9d4a-7c3193bed044", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:54.589Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "80a8ccc5-0146-48c2-9b30-dc0d04e8a3cf", "object": "some-name-pxc"} 2024-07-03T15:21:54.654Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "80a8ccc5-0146-48c2-9b30-dc0d04e8a3cf", "object": "some-name-pxc"} 2024-07-03T15:21:54.689Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "80a8ccc5-0146-48c2-9b30-dc0d04e8a3cf", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:21:59.836Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6df907a7-0e26-4033-91ff-c5a6ea7a99db", "object": "some-name-pxc"} 2024-07-03T15:21:59.904Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6df907a7-0e26-4033-91ff-c5a6ea7a99db", "object": "some-name-pxc"} 2024-07-03T15:21:59.968Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6df907a7-0e26-4033-91ff-c5a6ea7a99db", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:00.163Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "35e9c263-5259-4f27-b7ba-ccf494c725ec", "object": "some-name-pxc"} 2024-07-03T15:22:00.261Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "35e9c263-5259-4f27-b7ba-ccf494c725ec", "object": "some-name-pxc"} 2024-07-03T15:22:00.301Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "35e9c263-5259-4f27-b7ba-ccf494c725ec", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:05.147Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ee03d9cb-4e8e-4c5f-915a-983668c13b66", "object": "some-name-pxc"} 2024-07-03T15:22:05.243Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ee03d9cb-4e8e-4c5f-915a-983668c13b66", "object": "some-name-pxc"} 2024-07-03T15:22:05.281Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ee03d9cb-4e8e-4c5f-915a-983668c13b66", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:10.497Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d026b215-e94b-41a4-89ef-f100341b236d", "object": "some-name-pxc"} 2024-07-03T15:22:10.583Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d026b215-e94b-41a4-89ef-f100341b236d", "object": "some-name-pxc"} 2024-07-03T15:22:10.679Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d026b215-e94b-41a4-89ef-f100341b236d", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:15.820Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3383f85f-dac7-47b9-9ddd-0b74cb106524", "object": "some-name-pxc"} 2024-07-03T15:22:15.886Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3383f85f-dac7-47b9-9ddd-0b74cb106524", "object": "some-name-pxc"} 2024-07-03T15:22:15.924Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3383f85f-dac7-47b9-9ddd-0b74cb106524", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:21.069Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d10250bb-b4ba-4257-a67c-52ce9c0489c5", "object": "some-name-pxc"} 2024-07-03T15:22:21.149Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d10250bb-b4ba-4257-a67c-52ce9c0489c5", "object": "some-name-pxc"} 2024-07-03T15:22:21.196Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d10250bb-b4ba-4257-a67c-52ce9c0489c5", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:26.631Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cc673a5c-24e5-4674-ad12-8fc4e6c13abe", "object": "some-name-pxc"} 2024-07-03T15:22:26.847Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cc673a5c-24e5-4674-ad12-8fc4e6c13abe", "object": "some-name-pxc"} 2024-07-03T15:22:26.930Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "cc673a5c-24e5-4674-ad12-8fc4e6c13abe", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:32.082Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "012042f1-065d-4ec3-9edf-6a9f442b3d08", "object": "some-name-pxc"} 2024-07-03T15:22:32.130Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "012042f1-065d-4ec3-9edf-6a9f442b3d08", "object": "some-name-pxc"} 2024-07-03T15:22:32.166Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "012042f1-065d-4ec3-9edf-6a9f442b3d08", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:37.300Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3b1ae33b-9ccf-4e75-a90a-3d51385413f5", "object": "some-name-pxc"} 2024-07-03T15:22:37.354Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3b1ae33b-9ccf-4e75-a90a-3d51385413f5", "object": "some-name-pxc"} 2024-07-03T15:22:37.383Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3b1ae33b-9ccf-4e75-a90a-3d51385413f5", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:37.600Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fe0107c7-6898-4bf8-8ec5-033a2ba57fc6", "object": "some-name-pxc"} 2024-07-03T15:22:37.693Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fe0107c7-6898-4bf8-8ec5-033a2ba57fc6", "object": "some-name-pxc"} 2024-07-03T15:22:37.724Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fe0107c7-6898-4bf8-8ec5-033a2ba57fc6", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:42.571Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3e145fce-1afc-4ff8-a192-3daa07f94e8b", "object": "some-name-pxc"} 2024-07-03T15:22:42.637Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3e145fce-1afc-4ff8-a192-3daa07f94e8b", "object": "some-name-pxc"} 2024-07-03T15:22:42.671Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3e145fce-1afc-4ff8-a192-3daa07f94e8b", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:47.844Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2e91ab67-a59b-41f2-8954-1395e0f97c17", "object": "some-name-pxc"} 2024-07-03T15:22:47.893Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2e91ab67-a59b-41f2-8954-1395e0f97c17", "object": "some-name-pxc"} 2024-07-03T15:22:47.923Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2e91ab67-a59b-41f2-8954-1395e0f97c17", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:53.054Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4aae4efd-304e-4f9b-a65f-c25efea68d93", "object": "some-name-pxc"} 2024-07-03T15:22:53.120Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4aae4efd-304e-4f9b-a65f-c25efea68d93", "object": "some-name-pxc"} 2024-07-03T15:22:53.150Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4aae4efd-304e-4f9b-a65f-c25efea68d93", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:22:58.275Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4fd6197e-6027-4350-ad40-8fe32b4edb63", "object": "some-name-pxc"} 2024-07-03T15:22:58.493Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4fd6197e-6027-4350-ad40-8fe32b4edb63", "object": "some-name-pxc"} 2024-07-03T15:22:58.626Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4fd6197e-6027-4350-ad40-8fe32b4edb63", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:23:03.930Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9003a00b-41c1-46ee-a0a4-5eb0960cb7a1", "object": "some-name-pxc"} 2024-07-03T15:23:04.021Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9003a00b-41c1-46ee-a0a4-5eb0960cb7a1", "object": "some-name-pxc"} 2024-07-03T15:23:04.057Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9003a00b-41c1-46ee-a0a4-5eb0960cb7a1", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:23:09.274Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b8cc4eba-3b3f-4054-911b-e401dd9ca8bd", "object": "some-name-pxc"} 2024-07-03T15:23:09.334Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b8cc4eba-3b3f-4054-911b-e401dd9ca8bd", "object": "some-name-pxc"} 2024-07-03T15:23:09.363Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b8cc4eba-3b3f-4054-911b-e401dd9ca8bd", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:23:14.500Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07dedcde-7946-4d08-be69-821b6130e959", "object": "some-name-pxc"} 2024-07-03T15:23:14.580Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07dedcde-7946-4d08-be69-821b6130e959", "object": "some-name-pxc"} 2024-07-03T15:23:14.607Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "07dedcde-7946-4d08-be69-821b6130e959", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:23:14.746Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ea7f0f4f-64f9-41cd-9820-6f5070f88ddd", "object": "some-name-pxc"} 2024-07-03T15:23:14.800Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ea7f0f4f-64f9-41cd-9820-6f5070f88ddd", "object": "some-name-pxc"} 2024-07-03T15:23:14.836Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ea7f0f4f-64f9-41cd-9820-6f5070f88ddd", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:23:19.745Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f1e41dce-d920-4549-9518-af3d0a88935d", "object": "some-name-pxc"} 2024-07-03T15:23:19.847Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f1e41dce-d920-4549-9518-af3d0a88935d", "object": "some-name-pxc"} 2024-07-03T15:23:19.946Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f1e41dce-d920-4549-9518-af3d0a88935d", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:23:25.269Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f0b3170f-48e9-494d-b508-11886f350738", "object": "some-name-pxc"} 2024-07-03T15:23:25.391Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f0b3170f-48e9-494d-b508-11886f350738", "object": "some-name-pxc"} 2024-07-03T15:23:25.490Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f0b3170f-48e9-494d-b508-11886f350738", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:23:30.771Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "707e5b92-39c3-44a2-9305-0536e444b516", "object": "some-name-pxc"} 2024-07-03T15:23:30.840Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "707e5b92-39c3-44a2-9305-0536e444b516", "object": "some-name-pxc"} 2024-07-03T15:23:30.867Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "707e5b92-39c3-44a2-9305-0536e444b516", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:24:37.896Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "707e5b92-39c3-44a2-9305-0536e444b516", "err": "failed to connect to pod some-name-pxc-2: dial tcp: lookup some-name-pxc-2.some-name-pxc.users-19584: i/o timeout"} 2024-07-03T15:24:38.024Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "026a39be-c85e-4df3-b228-e4a3d838cbe8", "object": "some-name-pxc"} 2024-07-03T15:24:38.174Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "026a39be-c85e-4df3-b228-e4a3d838cbe8", "object": "some-name-pxc"} 2024-07-03T15:24:38.239Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "026a39be-c85e-4df3-b228-e4a3d838cbe8", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:24:54.413Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3c1de311-a17e-4b06-be35-63ebed1495f1", "object": "some-name-pxc"} 2024-07-03T15:24:54.598Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3c1de311-a17e-4b06-be35-63ebed1495f1", "object": "some-name-pxc"} 2024-07-03T15:24:54.645Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3c1de311-a17e-4b06-be35-63ebed1495f1", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:24:59.403Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a900a313-f8b1-48ff-b464-129fa2c6a852", "object": "some-name-pxc"} 2024-07-03T15:24:59.467Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a900a313-f8b1-48ff-b464-129fa2c6a852", "object": "some-name-pxc"} 2024-07-03T15:24:59.510Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a900a313-f8b1-48ff-b464-129fa2c6a852", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:04.693Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "efaca1bc-1ddc-419c-bfe7-eb130214f410", "object": "some-name-pxc"} 2024-07-03T15:25:04.770Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "efaca1bc-1ddc-419c-bfe7-eb130214f410", "object": "some-name-pxc"} 2024-07-03T15:25:04.812Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "efaca1bc-1ddc-419c-bfe7-eb130214f410", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:04.959Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7ad6562e-d675-4c15-bfe9-98e1e5a879d5", "object": "some-name-pxc"} 2024-07-03T15:25:05.038Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7ad6562e-d675-4c15-bfe9-98e1e5a879d5", "object": "some-name-pxc"} 2024-07-03T15:25:05.092Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "7ad6562e-d675-4c15-bfe9-98e1e5a879d5", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:09.964Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "89fd7200-3d0e-454f-ae14-523981dd47d6", "object": "some-name-pxc"} 2024-07-03T15:25:10.070Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "89fd7200-3d0e-454f-ae14-523981dd47d6", "object": "some-name-pxc"} 2024-07-03T15:25:10.124Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "89fd7200-3d0e-454f-ae14-523981dd47d6", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:15.279Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "162e2bef-4f98-4b8d-812d-b4c42e4f511b", "object": "some-name-pxc"} 2024-07-03T15:25:15.336Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "162e2bef-4f98-4b8d-812d-b4c42e4f511b", "object": "some-name-pxc"} 2024-07-03T15:25:15.373Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "162e2bef-4f98-4b8d-812d-b4c42e4f511b", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:20.498Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d1f8cd31-2c2c-4071-a2a8-a1244240e52c", "object": "some-name-pxc"} 2024-07-03T15:25:20.645Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d1f8cd31-2c2c-4071-a2a8-a1244240e52c", "object": "some-name-pxc"} 2024-07-03T15:25:20.695Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "d1f8cd31-2c2c-4071-a2a8-a1244240e52c", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:26.299Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fe4c8da5-17ae-4416-ba34-3251872a4448", "object": "some-name-pxc"} 2024-07-03T15:25:26.946Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fe4c8da5-17ae-4416-ba34-3251872a4448", "object": "some-name-pxc"} 2024-07-03T15:25:27.069Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fe4c8da5-17ae-4416-ba34-3251872a4448", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:32.288Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a414aa48-bd76-4aab-8695-06e3b94e37b9", "object": "some-name-pxc"} 2024-07-03T15:25:32.380Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a414aa48-bd76-4aab-8695-06e3b94e37b9", "object": "some-name-pxc"} 2024-07-03T15:25:32.423Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a414aa48-bd76-4aab-8695-06e3b94e37b9", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:32.591Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f229e7fb-fe30-4941-b9c9-26e14df1e4be", "object": "some-name-pxc"} 2024-07-03T15:25:32.669Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f229e7fb-fe30-4941-b9c9-26e14df1e4be", "object": "some-name-pxc"} 2024-07-03T15:25:32.711Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f229e7fb-fe30-4941-b9c9-26e14df1e4be", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:37.586Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5bde3d41-e360-4540-a9ec-a5199bbb1331", "object": "some-name-pxc"} 2024-07-03T15:25:37.659Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5bde3d41-e360-4540-a9ec-a5199bbb1331", "object": "some-name-pxc"} 2024-07-03T15:25:37.696Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5bde3d41-e360-4540-a9ec-a5199bbb1331", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:42.918Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5b19b855-2a25-4a48-af42-240650f2f1a1", "object": "some-name-pxc"} 2024-07-03T15:25:42.993Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5b19b855-2a25-4a48-af42-240650f2f1a1", "object": "some-name-pxc"} 2024-07-03T15:25:43.030Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "5b19b855-2a25-4a48-af42-240650f2f1a1", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:43.265Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ccdd5423-e533-4997-9ab6-cde8b5843236", "object": "some-name-pxc"} 2024-07-03T15:25:43.339Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ccdd5423-e533-4997-9ab6-cde8b5843236", "object": "some-name-pxc"} 2024-07-03T15:25:43.375Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "ccdd5423-e533-4997-9ab6-cde8b5843236", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:48.255Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "77f536e4-354f-4bad-a17b-a795b5826dea", "object": "some-name-pxc"} 2024-07-03T15:25:48.324Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "77f536e4-354f-4bad-a17b-a795b5826dea", "object": "some-name-pxc"} 2024-07-03T15:25:48.357Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "77f536e4-354f-4bad-a17b-a795b5826dea", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:53.536Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2193897d-68b5-4388-bd41-39190483c7bf", "object": "some-name-pxc"} 2024-07-03T15:25:53.598Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2193897d-68b5-4388-bd41-39190483c7bf", "object": "some-name-pxc"} 2024-07-03T15:25:53.646Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2193897d-68b5-4388-bd41-39190483c7bf", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:25:58.916Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2fecb9a2-2dbd-4864-8745-cc1b5caeb852", "object": "some-name-pxc"} 2024-07-03T15:25:58.996Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2fecb9a2-2dbd-4864-8745-cc1b5caeb852", "object": "some-name-pxc"} 2024-07-03T15:25:59.036Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "2fecb9a2-2dbd-4864-8745-cc1b5caeb852", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:04.228Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "405d21aa-acb1-4b7c-9535-f1184fbb16ea", "user": "monitor"} 2024-07-03T15:26:04.249Z INFO User password updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "405d21aa-acb1-4b7c-9535-f1184fbb16ea", "user": "monitor"} 2024-07-03T15:26:04.262Z INFO MySQL init secret updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "405d21aa-acb1-4b7c-9535-f1184fbb16ea", "secret": "some-name-mysql-init", "user": "monitor"} 2024-07-03T15:26:04.280Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "405d21aa-acb1-4b7c-9535-f1184fbb16ea", "user": "monitor"} 2024-07-03T15:26:04.280Z INFO Proxy pods will be restarted {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "405d21aa-acb1-4b7c-9535-f1184fbb16ea", "last-applied-secret": "81935ecf30ae523a6ee69582354f9488d47dbeb0f8c7c572af96870f2024284d"} 2024-07-03T15:26:04.283Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "405d21aa-acb1-4b7c-9535-f1184fbb16ea", "object": "some-name-pxc"} 2024-07-03T15:26:04.349Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "405d21aa-acb1-4b7c-9535-f1184fbb16ea", "object": "some-name-pxc"} 2024-07-03T15:26:04.391Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "405d21aa-acb1-4b7c-9535-f1184fbb16ea", "object": "some-name-haproxy"} 2024-07-03T15:26:04.441Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "405d21aa-acb1-4b7c-9535-f1184fbb16ea", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:04.893Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "57a3a52b-aa1a-49c7-9785-7a92370568fb", "object": "some-name-pxc"} 2024-07-03T15:26:04.969Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "57a3a52b-aa1a-49c7-9785-7a92370568fb", "object": "some-name-pxc"} 2024-07-03T15:26:05.006Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "57a3a52b-aa1a-49c7-9785-7a92370568fb", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:09.853Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4530db41-194f-4100-8fde-5b6f622a738f", "object": "some-name-pxc"} 2024-07-03T15:26:09.904Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4530db41-194f-4100-8fde-5b6f622a738f", "object": "some-name-pxc"} 2024-07-03T15:26:09.943Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "4530db41-194f-4100-8fde-5b6f622a738f", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:10.182Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f99ccca3-d4fb-42db-a073-22ef4c6f6a1f", "object": "some-name-pxc"} 2024-07-03T15:26:10.270Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f99ccca3-d4fb-42db-a073-22ef4c6f6a1f", "object": "some-name-pxc"} 2024-07-03T15:26:10.294Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "f99ccca3-d4fb-42db-a073-22ef4c6f6a1f", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:15.175Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6e0c026f-10a9-4c53-bb5d-8e2dfd52b42c", "object": "some-name-pxc"} 2024-07-03T15:26:15.257Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6e0c026f-10a9-4c53-bb5d-8e2dfd52b42c", "object": "some-name-pxc"} 2024-07-03T15:26:15.329Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "6e0c026f-10a9-4c53-bb5d-8e2dfd52b42c", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:20.564Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "37321815-8186-4469-92cf-23e15ce0878a", "object": "some-name-pxc"} 2024-07-03T15:26:20.641Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "37321815-8186-4469-92cf-23e15ce0878a", "object": "some-name-pxc"} 2024-07-03T15:26:20.675Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "37321815-8186-4469-92cf-23e15ce0878a", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:21.886Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fc203e8a-db3f-4ce9-ba65-4d1173012851", "object": "some-name-pxc"} 2024-07-03T15:26:21.958Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fc203e8a-db3f-4ce9-ba65-4d1173012851", "object": "some-name-pxc"} 2024-07-03T15:26:22.000Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "fc203e8a-db3f-4ce9-ba65-4d1173012851", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:27.023Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "551c6f8b-b655-4a94-b49f-243a98c5dc41", "object": "some-name-pxc"} 2024-07-03T15:26:27.184Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "551c6f8b-b655-4a94-b49f-243a98c5dc41", "object": "some-name-pxc"} 2024-07-03T15:26:27.297Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "551c6f8b-b655-4a94-b49f-243a98c5dc41", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:27.527Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c692ec4e-3e7b-40e8-812f-ab24c7ffd391", "object": "some-name-pxc"} 2024-07-03T15:26:27.585Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c692ec4e-3e7b-40e8-812f-ab24c7ffd391", "object": "some-name-pxc"} 2024-07-03T15:26:27.630Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "c692ec4e-3e7b-40e8-812f-ab24c7ffd391", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:32.527Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b8d5976f-ff4d-4858-b059-34399cf25b5f", "object": "some-name-pxc"} 2024-07-03T15:26:32.599Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b8d5976f-ff4d-4858-b059-34399cf25b5f", "object": "some-name-pxc"} 2024-07-03T15:26:32.639Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "b8d5976f-ff4d-4858-b059-34399cf25b5f", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:53.828Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8ce11170-0cb2-430d-8192-5333611f7639", "object": "some-name-pxc"} 2024-07-03T15:26:53.937Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8ce11170-0cb2-430d-8192-5333611f7639", "object": "some-name-pxc"} 2024-07-03T15:26:53.968Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "8ce11170-0cb2-430d-8192-5333611f7639", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:54.177Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3b30cc8a-06b1-45ee-98ea-074da192cbcd", "object": "some-name-pxc"} 2024-07-03T15:26:54.260Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3b30cc8a-06b1-45ee-98ea-074da192cbcd", "object": "some-name-pxc"} 2024-07-03T15:26:54.294Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3b30cc8a-06b1-45ee-98ea-074da192cbcd", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:26:59.171Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "55db73e3-2472-4221-b686-93f7de87ea73", "object": "some-name-pxc"} 2024-07-03T15:26:59.275Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "55db73e3-2472-4221-b686-93f7de87ea73", "object": "some-name-pxc"} 2024-07-03T15:26:59.373Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "55db73e3-2472-4221-b686-93f7de87ea73", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:27:04.597Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e9a6a669-5328-498c-92fd-5907959a1f0f", "object": "some-name-pxc"} 2024-07-03T15:27:04.713Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e9a6a669-5328-498c-92fd-5907959a1f0f", "object": "some-name-pxc"} 2024-07-03T15:27:04.752Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "e9a6a669-5328-498c-92fd-5907959a1f0f", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:27:09.925Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "70df1e7b-bf6a-4e8f-a1c1-1aa975ff5b7e", "object": "some-name-pxc"} 2024-07-03T15:27:09.998Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "70df1e7b-bf6a-4e8f-a1c1-1aa975ff5b7e", "object": "some-name-pxc"} 2024-07-03T15:27:10.087Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "70df1e7b-bf6a-4e8f-a1c1-1aa975ff5b7e", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:27:15.349Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3a9286df-41a8-4715-a5f9-6eb9bccafe89", "object": "some-name-pxc"} 2024-07-03T15:27:15.444Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3a9286df-41a8-4715-a5f9-6eb9bccafe89", "object": "some-name-pxc"} 2024-07-03T15:27:15.484Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "3a9286df-41a8-4715-a5f9-6eb9bccafe89", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:27:15.693Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "bf3d041d-604d-4de0-b8e9-8c2f6d0bc27b", "object": "some-name-pxc"} 2024-07-03T15:27:15.762Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "bf3d041d-604d-4de0-b8e9-8c2f6d0bc27b", "object": "some-name-pxc"} 2024-07-03T15:27:15.833Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "bf3d041d-604d-4de0-b8e9-8c2f6d0bc27b", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:27:20.675Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a30868b0-a2b7-4f40-9fcd-7e3da614f5c5", "object": "some-name-pxc"} 2024-07-03T15:27:20.744Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a30868b0-a2b7-4f40-9fcd-7e3da614f5c5", "object": "some-name-pxc"} 2024-07-03T15:27:20.791Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "a30868b0-a2b7-4f40-9fcd-7e3da614f5c5", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} 2024-07-03T15:27:26.789Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9a6b30c5-eb2a-4329-966d-6a688d1c39c1", "object": "some-name-pxc"} 2024-07-03T15:27:27.096Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9a6b30c5-eb2a-4329-966d-6a688d1c39c1", "object": "some-name-pxc"} 2024-07-03T15:27:27.159Z INFO HAProxy replicas service {"controller": "pxc-controller", "namespace": "users-19584", "name": "some-name", "reconcileID": "9a6b30c5-eb2a-4329-966d-6a688d1c39c1", "service": "some-name-haproxy-replicas", "expose": {"enabled":true}} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1250 [mysql] 2024/07/03 15:26:20 connection.go:49: unexpected EOF sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n users-19584 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.GDc6q1tLLJ ++ mktemp + local LAST_ERR=/tmp/tmp.oAgB56JMGb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GDc6q1tLLJ perconaxtradbcluster.pxc.percona.com "some-name" deleted + cat /tmp/tmp.oAgB56JMGb + rm /tmp/tmp.GDc6q1tLLJ /tmp/tmp.oAgB56JMGb + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.194gTaFoIm ++ mktemp + local LAST_ERR=/tmp/tmp.J6heVi2OXE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.194gTaFoIm No resources found + cat /tmp/tmp.J6heVi2OXE + rm /tmp/tmp.194gTaFoIm /tmp/tmp.J6heVi2OXE + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.zRgPDzndE9 ++ mktemp + local LAST_ERR=/tmp/tmp.UipP3gBY8T + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zRgPDzndE9 No resources found + cat /tmp/tmp.UipP3gBY8T + rm /tmp/tmp.zRgPDzndE9 /tmp/tmp.UipP3gBY8T + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.FCLopY5iKw ++ mktemp + local LAST_ERR=/tmp/tmp.pMYG56VnBq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FCLopY5iKw validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.pMYG56VnBq + rm /tmp/tmp.FCLopY5iKw /tmp/tmp.pMYG56VnBq + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace users-19584 + rm -rf /tmp/tmp.q8i5quND1N + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.A9tD9UB9wo + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- ++ mktemp + local LAST_OUT=/tmp/tmp.0wUHVv4kLi ++ mktemp + local LAST_ERR=/tmp/tmp.EkgR85EdbO + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.8GkXiHdyPH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace users-19584 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator