Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/logs/cross-site-proxysql-8-4.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + main + local source_cluster=cross-site-source + local replica_cluster=cross-site-replica + desc 'Create source cluster' + set +o xtrace ----------------------------------------------------------------------------------- Create source cluster ----------------------------------------------------------------------------------- + create_infra cross-site-proxysql-28035 + local ns=cross-site-proxysql-28035 + '[' -n pxc-operator ']' + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + [[ -z cross-site-proxysql-30334 ]] + [[ -z cross-site-source ]] + kubectl_bin patch pxc -n cross-site-proxysql-30334 cross-site-source --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.hyY5IQZcjg ++ mktemp + local LAST_ERR=/tmp/tmp.1qECrvcoD1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc -n cross-site-proxysql-30334 cross-site-source --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hyY5IQZcjg perconaxtradbcluster.pxc.percona.com/cross-site-source patched + cat /tmp/tmp.1qECrvcoD1 + rm /tmp/tmp.hyY5IQZcjg /tmp/tmp.1qECrvcoD1 + return 0 + read -r pns name + [[ -z cross-site-proxysql-replica-27699 ]] + [[ -z cross-site-replica ]] + kubectl_bin patch pxc -n cross-site-proxysql-replica-27699 cross-site-replica --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.A0a6Pp5Xwf ++ mktemp + local LAST_ERR=/tmp/tmp.FBrmZ4Z2kM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc -n cross-site-proxysql-replica-27699 cross-site-replica --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.A0a6Pp5Xwf perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + cat /tmp/tmp.FBrmZ4Z2kM + rm /tmp/tmp.A0a6Pp5Xwf /tmp/tmp.FBrmZ4Z2kM + return 0 + read -r pns name + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + [[ -z cross-site-proxysql-30334 ]] + [[ -z backup-minio-source ]] + kubectl_bin patch pxc-backup -n cross-site-proxysql-30334 backup-minio-source --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.1gc6grUn6d ++ mktemp + local LAST_ERR=/tmp/tmp.oDfZBLcdlk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc-backup -n cross-site-proxysql-30334 backup-minio-source --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1gc6grUn6d perconaxtradbclusterbackup.pxc.percona.com/backup-minio-source patched (no change) + cat /tmp/tmp.oDfZBLcdlk + rm /tmp/tmp.1gc6grUn6d /tmp/tmp.oDfZBLcdlk + return 0 + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found perconaxtradbcluster.pxc.percona.com "cross-site-source" deleted from cross-site-proxysql-30334 namespace perconaxtradbcluster.pxc.percona.com "cross-site-replica" deleted from cross-site-proxysql-replica-27699 namespace + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found perconaxtradbclusterbackup.pxc.percona.com "backup-minio-source" deleted from cross-site-proxysql-30334 namespace + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found perconaxtradbclusterrestore.pxc.percona.com "backup-minio" deleted from cross-site-proxysql-replica-27699 namespace + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + xargs kubectl delete ns + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.XES8cOlAjT ++ mktemp + local LAST_OUT=/tmp/tmp.LNnoS1Mqu8 ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.8Yo8L8QENY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.qvsFojuRzb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LNnoS1Mqu8 + cat /tmp/tmp.8Yo8L8QENY + rm /tmp/tmp.LNnoS1Mqu8 /tmp/tmp.8Yo8L8QENY + return 0 namespace "cross-site-proxysql-30334" deleted namespace "cross-site-proxysql-replica-27699" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XES8cOlAjT namespace "pxc-operator" deleted + cat /tmp/tmp.qvsFojuRzb + rm /tmp/tmp.XES8cOlAjT /tmp/tmp.qvsFojuRzb + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.D8R1U0Mbbl ++ mktemp + local LAST_ERR=/tmp/tmp.ksBkDydw7U + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.D8R1U0Mbbl namespace/pxc-operator created + cat /tmp/tmp.ksBkDydw7U + rm /tmp/tmp.D8R1U0Mbbl /tmp/tmp.ksBkDydw7U + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Xgv77vhHel +++ mktemp ++ local LAST_ERR=/tmp/tmp.8pBUXCeBZp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Xgv77vhHel ++ cat /tmp/tmp.8pBUXCeBZp ++ rm /tmp/tmp.Xgv77vhHel /tmp/tmp.8pBUXCeBZp ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.3KzwRafH8H ++ mktemp + local LAST_ERR=/tmp/tmp.04i2TP3Unu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3KzwRafH8H Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7" modified. + cat /tmp/tmp.04i2TP3Unu + rm /tmp/tmp.3KzwRafH8H /tmp/tmp.04i2TP3Unu + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.vtWvoSwbE6 ++ mktemp + local LAST_ERR=/tmp/tmp.5LpFeBV5L5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vtWvoSwbE6 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.5LpFeBV5L5 + rm /tmp/tmp.vtWvoSwbE6 /tmp/tmp.5LpFeBV5L5 + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.ymvzKiWYrd ++ mktemp + local LAST_ERR=/tmp/tmp.m5bEaa2lQB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ymvzKiWYrd customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.m5bEaa2lQB E0702 02:40:24.123795 26020 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusters.pxc.percona.com&resourceVersion=1782960020105077000&timeoutSeconds=356&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.ymvzKiWYrd /tmp/tmp.m5bEaa2lQB + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.0foweRZ8CE ++ mktemp + local LAST_ERR=/tmp/tmp.3Q11VBgipO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0foweRZ8CE customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.3Q11VBgipO E0702 02:40:26.396699 26463 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterbackups.pxc.percona.com&resourceVersion=1782960025105128000&timeoutSeconds=341&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.0foweRZ8CE /tmp/tmp.3Q11VBgipO + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.2S2ifGkdRu ++ mktemp + local LAST_ERR=/tmp/tmp.62QnJIOuwR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2S2ifGkdRu customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.62QnJIOuwR E0702 02:40:28.520711 26747 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterrestores.pxc.percona.com&resourceVersion=1782960025105128000&timeoutSeconds=329&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.2S2ifGkdRu /tmp/tmp.62QnJIOuwR + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.Z3DLjtCXOQ ++ mktemp + local LAST_ERR=/tmp/tmp.7aonypDKoY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Z3DLjtCXOQ clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.7aonypDKoY + rm /tmp/tmp.Z3DLjtCXOQ /tmp/tmp.7aonypDKoY + return 0 + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-operator.yaml + local LAST_OUT=/tmp/tmp.axwOVi6V5a ++ mktemp + local LAST_ERR=/tmp/tmp.iMYT3czJl4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.axwOVi6V5a deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.iMYT3czJl4 + rm /tmp/tmp.axwOVi6V5a /tmp/tmp.iMYT3czJl4 + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.idiKmvtkhn ++ mktemp + local LAST_ERR=/tmp/tmp.byxiwLN1lu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.idiKmvtkhn pod/percona-xtradb-cluster-operator-65b5d85575-9dbzk condition met + cat /tmp/tmp.byxiwLN1lu E0702 02:40:49.866638 30029 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-9dbzk&resourceVersion=1782960049504101000&timeoutSeconds=361&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.idiKmvtkhn /tmp/tmp.byxiwLN1lu + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.pGj48wUapm +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZFld7IC5wV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pGj48wUapm ++ cat /tmp/tmp.ZFld7IC5wV ++ rm /tmp/tmp.pGj48wUapm /tmp/tmp.ZFld7IC5wV ++ return 0 + wait_pod percona-xtradb-cluster-operator-65b5d85575-9dbzk 480 pxc-operator + local pod=percona-xtradb-cluster-operator-65b5d85575-9dbzk + local max_retry=480 + local ns=pxc-operator ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo percona-xtradb-cluster-operator-65b5d85575-9dbzk + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-65b5d85575-9dbzk condition met E0702 02:40:56.440580 31436 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-9dbzk&resourceVersion=1782960054076950000&timeoutSeconds=337&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-65b5d85575-9dbzk to become Ready.Ok + sleep 3 + create_namespace cross-site-proxysql-28035 + local namespace=cross-site-proxysql-28035 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-proxysql-28035' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-proxysql-28035 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-proxysql-28035 + xargs kubectl delete ns + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.dAOcAGkZAo ++ mktemp + awk '{print$1}' + local LAST_OUT=/tmp/tmp.6oU8u3hoxU ++ mktemp + local LAST_ERR=/tmp/tmp.SHbZpdfbXP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-proxysql-28035 ++ mktemp + local LAST_ERR=/tmp/tmp.rrz3Zv5pbk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6oU8u3hoxU + cat /tmp/tmp.rrz3Zv5pbk + rm /tmp/tmp.6oU8u3hoxU /tmp/tmp.rrz3Zv5pbk + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-proxysql-28035 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-proxysql-28035 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.dAOcAGkZAo + cat /tmp/tmp.SHbZpdfbXP Error from server (NotFound): namespaces "cross-site-proxysql-28035" not found + rm /tmp/tmp.dAOcAGkZAo /tmp/tmp.SHbZpdfbXP + return 1 + : + wait_for_delete namespace/cross-site-proxysql-28035 + local res=namespace/cross-site-proxysql-28035 + echo -n 'waiting for namespace/cross-site-proxysql-28035 to be deleted' waiting for namespace/cross-site-proxysql-28035 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "cross-site-proxysql-28035" not found + desc 'create namespace cross-site-proxysql-28035' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-proxysql-28035 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-proxysql-28035 ++ mktemp + local LAST_OUT=/tmp/tmp.F5Gns8Wvng ++ mktemp + local LAST_ERR=/tmp/tmp.3TU8GB6hhy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cross-site-proxysql-28035 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.F5Gns8Wvng namespace/cross-site-proxysql-28035 created + cat /tmp/tmp.3TU8GB6hhy + rm /tmp/tmp.F5Gns8Wvng /tmp/tmp.3TU8GB6hhy + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.1AGwG4Gj3K +++ mktemp ++ local LAST_ERR=/tmp/tmp.N9uxd7HjFR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1AGwG4Gj3K ++ cat /tmp/tmp.N9uxd7HjFR ++ rm /tmp/tmp.1AGwG4Gj3K /tmp/tmp.N9uxd7HjFR ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-28035 ++ mktemp + local LAST_OUT=/tmp/tmp.iaKp0s3tv9 ++ mktemp + local LAST_ERR=/tmp/tmp.dhqqKSLePS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-28035 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iaKp0s3tv9 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7" modified. + cat /tmp/tmp.dhqqKSLePS + rm /tmp/tmp.iaKp0s3tv9 /tmp/tmp.dhqqKSLePS + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.GddM5lZVzT ++ mktemp + local LAST_ERR=/tmp/tmp.ECZnFc1Ajw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GddM5lZVzT secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.ECZnFc1Ajw + rm /tmp/tmp.GddM5lZVzT /tmp/tmp.ECZnFc1Ajw + return 0 + start_minio + deploy_helm cross-site-proxysql-28035 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Thu Jul 2 02:41:43 2026 NAMESPACE: cross-site-proxysql-28035 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.cross-site-proxysql-28035.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace cross-site-proxysql-28035 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace cross-site-proxysql-28035 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace cross-site-proxysql-28035 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace cross-site-proxysql-28035 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qwmpYTKWME +++ mktemp ++ local LAST_ERR=/tmp/tmp.ausw3bjVZk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qwmpYTKWME ++ cat /tmp/tmp.ausw3bjVZk ++ rm /tmp/tmp.qwmpYTKWME /tmp/tmp.ausw3bjVZk ++ return 0 + MINIO_POD=minio-service-5fd5489bdc-zcfvs + wait_pod minio-service-5fd5489bdc-zcfvs + local pod=minio-service-5fd5489bdc-zcfvs + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo minio-service-5fd5489bdc-zcfvs ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/minio-service-5fd5489bdc-zcfvs condition met E0702 02:42:47.325876 15401 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dminio-service-5fd5489bdc-zcfvs&resourceVersion=1782960164635780000&timeoutSeconds=350&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/minio-service-5fd5489bdc-zcfvs to become Ready.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.BB7AiBZND5 ++ mktemp + local LAST_ERR=/tmp/tmp.jPpzXg2tb4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BB7AiBZND5 make_bucket: operator-testing make_bucket: operator-testing pod "aws-cli" deleted from cross-site-proxysql-28035 namespace + cat /tmp/tmp.jPpzXg2tb4 All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_cross-site-proxysql-28035 + rm /tmp/tmp.BB7AiBZND5 /tmp/tmp.jPpzXg2tb4 + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ux9Yq55RK3 ++ mktemp + local LAST_ERR=/tmp/tmp.OaIAOCAiRq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ux9Yq55RK3 secret/minio-secret unchanged secret/aws-s3-secret unchanged secret/do-spaces-secret unchanged secret/gcp-cs-secret unchanged secret/azure-secret unchanged + cat /tmp/tmp.OaIAOCAiRq + rm /tmp/tmp.ux9Yq55RK3 /tmp/tmp.OaIAOCAiRq + return 0 + spinup_pxc cross-site-source /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-source.yml + local cluster=cross-site-source + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-source.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.2NA0cKwyS2 ++ mktemp + local LAST_ERR=/tmp/tmp.JgsTZMdjY6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2NA0cKwyS2 secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.JgsTZMdjY6 + rm /tmp/tmp.2NA0cKwyS2 /tmp/tmp.JgsTZMdjY6 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local pvc_name= + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-proxysql-28035~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.cIEf01i4dw + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' ++ mktemp + local LAST_ERR=/tmp/tmp.qgsKrMMv45 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cIEf01i4dw deployment.apps/pxc-client created + cat /tmp/tmp.qgsKrMMv45 + rm /tmp/tmp.cIEf01i4dw /tmp/tmp.qgsKrMMv45 + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-source.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-source.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-source.yml '' + kubectl_bin apply -f - + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-source.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-proxysql-28035~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#' ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-source.yml + local LAST_OUT=/tmp/tmp.rxVm8Pk3bp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + local LAST_ERR=/tmp/tmp.5NUm88EDPN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rxVm8Pk3bp perconaxtradbcluster.pxc.percona.com/cross-site-source created + cat /tmp/tmp.5NUm88EDPN + rm /tmp/tmp.rxVm8Pk3bp /tmp/tmp.5NUm88EDPN + return 0 + desc 'waiting pods to be running for cross-site-source' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for cross-site-source ----------------------------------------------------------------------------------- ++ get_proxy cross-site-source ++ local target_cluster=cross-site-source +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6BMs4oQrd4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CMbikzL47j +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.6BMs4oQrd4 +++ cat /tmp/tmp.CMbikzL47j +++ rm /tmp/tmp.6BMs4oQrd4 /tmp/tmp.CMbikzL47j +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IrqjsTHO7Z ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JqZ2zXnj18 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.IrqjsTHO7Z +++ cat /tmp/tmp.JqZ2zXnj18 +++ rm /tmp/tmp.IrqjsTHO7Z /tmp/tmp.JqZ2zXnj18 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo cross-site-source-proxysql ++ return + local proxy=cross-site-source-proxysql ++ get_proxy_size cross-site-source ++ local cluster=cross-site-source ++ local size=0 +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.287iE3ZoKc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xOMiq8S9i4 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.287iE3ZoKc +++ cat /tmp/tmp.xOMiq8S9i4 +++ rm /tmp/tmp.287iE3ZoKc /tmp/tmp.xOMiq8S9i4 +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.l9RjmdQAHq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RoXtwHHLQ4 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.l9RjmdQAHq +++ cat /tmp/tmp.RoXtwHHLQ4 +++ rm /tmp/tmp.l9RjmdQAHq /tmp/tmp.RoXtwHHLQ4 +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VJqTExdyou ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wEjOSeLmB7 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.VJqTExdyou +++ cat /tmp/tmp.wEjOSeLmB7 +++ rm /tmp/tmp.VJqTExdyou /tmp/tmp.wEjOSeLmB7 +++ return 0 ++ size=3 ++ echo 3 + local proxy_size=3 + wait_for_running cross-site-source-pxc 3 + local name=cross-site-source-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z cross-site-source-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for cross-site-source-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for cross-site-source-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-0 480 + local pod=cross-site-source-pxc-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo cross-site-source-pxc-0 + local container=pxc + set +o xtrace pod/cross-site-source-pxc-0 condition met waiting for pod/cross-site-source-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-1 480 + local pod=cross-site-source-pxc-1 + local max_retry=480 + local ns= ++ echo cross-site-source-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-source-pxc-1 condition met waiting for pod/cross-site-source-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-2 480 + local pod=cross-site-source-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo cross-site-source-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/cross-site-source-pxc-2 condition met waiting for pod/cross-site-source-pxc-2 to become Ready.Ok + wait_for_running cross-site-source-proxysql 3 + local name=cross-site-source-proxysql + local size=3 + local last_pod + local max_retry=480 + [[ -z cross-site-source-proxysql ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for cross-site-source-proxysql' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for cross-site-source-proxysql ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-proxysql-0 480 + local pod=cross-site-source-proxysql-0 + local max_retry=480 + local ns= ++ echo cross-site-source-proxysql-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace pod/cross-site-source-proxysql-0 condition met E0702 02:47:18.064510 6875 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcross-site-source-proxysql-0&resourceVersion=1782960437885025000&timeoutSeconds=472&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/cross-site-source-proxysql-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-proxysql-1 480 + local pod=cross-site-source-proxysql-1 + local max_retry=480 + local ns= ++ echo cross-site-source-proxysql-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/cross-site-source-proxysql-1 condition met E0702 02:47:24.687533 13468 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcross-site-source-proxysql-1&resourceVersion=1782960442885141000&timeoutSeconds=337&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/cross-site-source-proxysql-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-proxysql-2 480 + local pod=cross-site-source-proxysql-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo cross-site-source-proxysql-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace pod/cross-site-source-proxysql-2 condition met E0702 02:47:31.550178 20694 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcross-site-source-proxysql-2&resourceVersion=1782960447885153000&timeoutSeconds=385&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/cross-site-source-proxysql-2 to become Ready.Ok + wait_cluster_consistency cross-site-source 3 3 + local cluster_name=cross-site-source + local pxc_size=3 + local proxy_size=3 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/cross-site-source to be ready' waiting for pxc/cross-site-source to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Tdaw62ievM +++ mktemp ++ local LAST_ERR=/tmp/tmp.HmVOSzceGB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Tdaw62ievM ++ cat /tmp/tmp.HmVOSzceGB ++ rm /tmp/tmp.Tdaw62ievM /tmp/tmp.HmVOSzceGB ++ return 0 + state=ready ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z0bGzJqZcp +++ mktemp ++ local LAST_ERR=/tmp/tmp.jaIHAcgkBj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.z0bGzJqZcp ++ cat /tmp/tmp.jaIHAcgkBj ++ rm /tmp/tmp.z0bGzJqZcp /tmp/tmp.jaIHAcgkBj ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.wIAEOWTMXk ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.6yMHmE24l3 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.wIAEOWTMXk +++++ cat /tmp/tmp.6yMHmE24l3 +++++ rm /tmp/tmp.wIAEOWTMXk /tmp/tmp.6yMHmE24l3 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.R1K8Cr7V3k ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.kvwx7tWUbW +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.R1K8Cr7V3k +++++ cat /tmp/tmp.kvwx7tWUbW +++++ rm /tmp/tmp.R1K8Cr7V3k /tmp/tmp.kvwx7tWUbW +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GFWxxDAzfg +++ mktemp ++ local LAST_ERR=/tmp/tmp.U8xKSkYiun ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GFWxxDAzfg ++ cat /tmp/tmp.U8xKSkYiun ++ rm /tmp/tmp.GFWxxDAzfg /tmp/tmp.U8xKSkYiun ++ return 0 + proxy_ready=3 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 3 == \3 ]] + echo + return + sleep 10 ++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DrXH87NjDB +++ mktemp ++ local LAST_ERR=/tmp/tmp.VV8Pb2T6ae ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DrXH87NjDB ++ cat /tmp/tmp.VV8Pb2T6ae ++ rm /tmp/tmp.DrXH87NjDB /tmp/tmp.VV8Pb2T6ae ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q3l6wdbX3e +++ mktemp ++ local LAST_ERR=/tmp/tmp.P1LzjYeEQ9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.q3l6wdbX3e ++ cat /tmp/tmp.P1LzjYeEQ9 ++ rm /tmp/tmp.q3l6wdbX3e /tmp/tmp.P1LzjYeEQ9 ++ return 0 + client_pod=pxc-client-56fd5498cd-c947g + wait_pod pxc-client-56fd5498cd-c947g + local pod=pxc-client-56fd5498cd-c947g + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-c947g ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 02:48:08.575278 26663 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782960487061464000&timeoutSeconds=421&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data cross-site-source 100500 no-proxy -P3306 + local cluster=cross-site-source + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password cross-site-source root ++ local cluster=cross-site-source ++ local user=root +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.l5KVwCn2jb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jVlUfOxstR +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.l5KVwCn2jb +++ cat /tmp/tmp.jVlUfOxstR +++ rm /tmp/tmp.l5KVwCn2jb /tmp/tmp.jVlUfOxstR +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nFXxyRFdOV +++ mktemp ++ local LAST_ERR=/tmp/tmp.9UHYJY6fKH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nFXxyRFdOV ++ cat /tmp/tmp.9UHYJY6fKH ++ rm /tmp/tmp.nFXxyRFdOV /tmp/tmp.9UHYJY6fKH ++ return 0 + root_pass=root_password ++ get_proxy cross-site-source ++ local target_cluster=cross-site-source +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.wREXDXqyJZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tPgMyJEsaE +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.wREXDXqyJZ +++ cat /tmp/tmp.tPgMyJEsaE +++ rm /tmp/tmp.wREXDXqyJZ /tmp/tmp.tPgMyJEsaE +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Y2x3sOb8ZQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Rim0kHUKZU +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Y2x3sOb8ZQ +++ cat /tmp/tmp.Rim0kHUKZU +++ rm /tmp/tmp.Y2x3sOb8ZQ /tmp/tmp.Rim0kHUKZU +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo cross-site-source-proxysql ++ return + proxy=cross-site-source-proxysql + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h cross-site-source-pxc-0.cross-site-source-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h cross-site-source-pxc-0.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GXiaXclNmX +++ mktemp ++ local LAST_ERR=/tmp/tmp.DbNs2nLbm3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GXiaXclNmX ++ cat /tmp/tmp.DbNs2nLbm3 ++ rm /tmp/tmp.GXiaXclNmX /tmp/tmp.DbNs2nLbm3 ++ return 0 + client_pod=pxc-client-56fd5498cd-c947g + wait_pod pxc-client-56fd5498cd-c947g + local pod=pxc-client-56fd5498cd-c947g + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-56fd5498cd-c947g + local container= + set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 02:48:19.771307 28235 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782960498780048000&timeoutSeconds=450&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data cross-site-source select-1 no-proxy -P3306 + local cluster=cross-site-source + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password cross-site-source root ++ local cluster=cross-site-source ++ local user=root +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QrArLpu0Ou ++++ mktemp +++ local LAST_ERR=/tmp/tmp.65m96J5jGV +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QrArLpu0Ou +++ cat /tmp/tmp.65m96J5jGV +++ rm /tmp/tmp.QrArLpu0Ou /tmp/tmp.65m96J5jGV +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.O5yDRbQxTj +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z3qLQ7rcz4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.O5yDRbQxTj ++ cat /tmp/tmp.Z3qLQ7rcz4 ++ rm /tmp/tmp.O5yDRbQxTj /tmp/tmp.Z3qLQ7rcz4 ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size cross-site-source ++ local cluster=cross-site-source ++ local size=0 +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lOCQNRf3Ji ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1OGcbqKbpF +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.lOCQNRf3Ji +++ cat /tmp/tmp.1OGcbqKbpF +++ rm /tmp/tmp.lOCQNRf3Ji /tmp/tmp.1OGcbqKbpF +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h cross-site-source-pxc-0.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h cross-site-source-pxc-0.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8HXQSq9NL2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.CkOrXKlxrA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8HXQSq9NL2 ++ cat /tmp/tmp.CkOrXKlxrA ++ rm /tmp/tmp.8HXQSq9NL2 /tmp/tmp.CkOrXKlxrA ++ return 0 + client_pod=pxc-client-56fd5498cd-c947g + wait_pod pxc-client-56fd5498cd-c947g + local pod=pxc-client-56fd5498cd-c947g + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-56fd5498cd-c947g + local container= + set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 02:49:09.316087 2976 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782960546146290000&timeoutSeconds=586&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1.sql /tmp/tmp.xyHcWb7plX/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h cross-site-source-pxc-1.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h cross-site-source-pxc-1.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h cross-site-source-pxc-1.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h cross-site-source-pxc-1.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.INL5e7R4Hr +++ mktemp ++ local LAST_ERR=/tmp/tmp.p2Hb4YzeBB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.INL5e7R4Hr ++ cat /tmp/tmp.p2Hb4YzeBB ++ rm /tmp/tmp.INL5e7R4Hr /tmp/tmp.p2Hb4YzeBB ++ return 0 + client_pod=pxc-client-56fd5498cd-c947g + wait_pod pxc-client-56fd5498cd-c947g + local pod=pxc-client-56fd5498cd-c947g + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-56fd5498cd-c947g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 02:49:22.513571 5140 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782960560066819000&timeoutSeconds=340&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1.sql /tmp/tmp.xyHcWb7plX/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h cross-site-source-pxc-2.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h cross-site-source-pxc-2.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h cross-site-source-pxc-2.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h cross-site-source-pxc-2.cross-site-source-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V1Gizg63ci +++ mktemp ++ local LAST_ERR=/tmp/tmp.2RGvypRLCZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.V1Gizg63ci ++ cat /tmp/tmp.2RGvypRLCZ ++ rm /tmp/tmp.V1Gizg63ci /tmp/tmp.2RGvypRLCZ ++ return 0 + client_pod=pxc-client-56fd5498cd-c947g + wait_pod pxc-client-56fd5498cd-c947g + local pod=pxc-client-56fd5498cd-c947g + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-c947g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 02:49:39.767467 7708 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782960577001583000&timeoutSeconds=447&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1.sql /tmp/tmp.xyHcWb7plX/select-1.sql + is_keyring_plugin_in_use cross-site-source + local cluster=cross-site-source + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + kubectl exec cross-site-source-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my + return 1 + wait_pxc_ready cross-site-source + local cluster=cross-site-source + local retry=0 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' + '[' ready = ready ']' + desc 'get main cluster services endpoints' + set +o xtrace ----------------------------------------------------------------------------------- get main cluster services endpoints ----------------------------------------------------------------------------------- ++ get_service_ip cross-site-source-pxc-0 ++ local service=cross-site-source-pxc-0 ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.57kkc9M4wQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.upLhZ19Vei +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.57kkc9M4wQ +++ cat /tmp/tmp.upLhZ19Vei +++ rm /tmp/tmp.57kkc9M4wQ /tmp/tmp.upLhZ19Vei +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H7Ui9aEhGZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.ImPlDekD2u ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.H7Ui9aEhGZ ++ cat /tmp/tmp.ImPlDekD2u ++ rm /tmp/tmp.H7Ui9aEhGZ /tmp/tmp.ImPlDekD2u ++ return 0 ++ return + source_endpoint0=34.118.239.56 ++ get_service_ip cross-site-source-pxc-1 ++ local service=cross-site-source-pxc-1 ++ grep -q NotFound ++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LS9QhJGlYO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nkFqbtd18R +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.LS9QhJGlYO +++ cat /tmp/tmp.nkFqbtd18R +++ rm /tmp/tmp.LS9QhJGlYO /tmp/tmp.nkFqbtd18R +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QaTAZtvhs2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.WI0Ly90PNH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QaTAZtvhs2 ++ cat /tmp/tmp.WI0Ly90PNH ++ rm /tmp/tmp.QaTAZtvhs2 /tmp/tmp.WI0Ly90PNH ++ return 0 ++ return + source_endpoint1=34.118.235.139 ++ get_service_ip cross-site-source-pxc-2 ++ local service=cross-site-source-pxc-2 ++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LpDD6Nrl6e ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Cwomz0FHxb +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.LpDD6Nrl6e +++ cat /tmp/tmp.Cwomz0FHxb +++ rm /tmp/tmp.LpDD6Nrl6e /tmp/tmp.Cwomz0FHxb +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.T7cvCh7jSz +++ mktemp ++ local LAST_ERR=/tmp/tmp.MuDBdlnZUl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.T7cvCh7jSz ++ cat /tmp/tmp.MuDBdlnZUl ++ rm /tmp/tmp.T7cvCh7jSz /tmp/tmp.MuDBdlnZUl ++ return 0 ++ return + source_endpoint2=34.118.228.242 ++ run_mysql 'SELECT @@hostname hostname;' '-h cross-site-source-proxysql -p3306 -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h cross-site-source-proxysql -p3306 -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HvUNWAUBtY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ukid4frHke +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.HvUNWAUBtY +++ cat /tmp/tmp.ukid4frHke +++ rm /tmp/tmp.HvUNWAUBtY /tmp/tmp.ukid4frHke +++ return 0 ++ client_pod=pxc-client-56fd5498cd-c947g ++ wait_pod pxc-client-56fd5498cd-c947g ++ local pod=pxc-client-56fd5498cd-c947g ++ local max_retry=480 ++ local ns= +++ echo pxc-client-56fd5498cd-c947g +++ grep -E '^(pxc|proxysql)$' +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ local container= ++ set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 02:50:16.523422 13214 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782960614522749000&timeoutSeconds=439&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + source_primary=cross-site-source-pxc-0 ++ get_service_ip cross-site-source-pxc-0 ++ local service=cross-site-source-pxc-0 ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4E577CYAbo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tUexjVL0fX +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.4E577CYAbo +++ cat /tmp/tmp.tUexjVL0fX +++ rm /tmp/tmp.4E577CYAbo /tmp/tmp.tUexjVL0fX +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hxPPvBTxPs +++ mktemp ++ local LAST_ERR=/tmp/tmp.P1t3LZALxE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hxPPvBTxPs ++ cat /tmp/tmp.P1t3LZALxE ++ rm /tmp/tmp.hxPPvBTxPs /tmp/tmp.P1t3LZALxE ++ return 0 ++ return + source_primary_endpoint=34.118.239.56 + desc 'patch source cluster with replicationChannels settings' + set +o xtrace ----------------------------------------------------------------------------------- patch source cluster with replicationChannels settings ----------------------------------------------------------------------------------- + kubectl_bin patch pxc cross-site-source --type=merge --patch '{"spec": {"pxc":{"replicationChannels": [{"name":"source_to_replica", "isSource": true}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.Bt0evc9BOP ++ mktemp + local LAST_ERR=/tmp/tmp.gGxeCF0zua + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-source --type=merge --patch '{"spec": {"pxc":{"replicationChannels": [{"name":"source_to_replica", "isSource": true}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Bt0evc9BOP perconaxtradbcluster.pxc.percona.com/cross-site-source patched + cat /tmp/tmp.gGxeCF0zua + rm /tmp/tmp.Bt0evc9BOP /tmp/tmp.gGxeCF0zua + return 0 + desc 'patch main cluster secrets with replication user' + set +o xtrace ----------------------------------------------------------------------------------- patch main cluster secrets with replication user ----------------------------------------------------------------------------------- + patch_replication_secret cross-site-source 3 3 + local cluster=cross-site-source + local pxc_size=3 + local proxy_size=3 ++ echo -n new_password ++ base64 + kubectl_bin patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.UTBpyh5kq1 ++ mktemp + local LAST_ERR=/tmp/tmp.xDIARLlThN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UTBpyh5kq1 secret/my-cluster-secrets patched + cat /tmp/tmp.xDIARLlThN + rm /tmp/tmp.UTBpyh5kq1 /tmp/tmp.xDIARLlThN + return 0 + sleep 15 + wait_cluster_consistency cross-site-source 3 3 + local cluster_name=cross-site-source + local pxc_size=3 + local proxy_size=3 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/cross-site-source to be ready' waiting for pxc/cross-site-source to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rW8a9ppuIp +++ mktemp ++ local LAST_ERR=/tmp/tmp.MHoJHj5rlZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rW8a9ppuIp ++ cat /tmp/tmp.MHoJHj5rlZ ++ rm /tmp/tmp.rW8a9ppuIp /tmp/tmp.MHoJHj5rlZ ++ return 0 + state=ready ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jLmsCaaCoP +++ mktemp ++ local LAST_ERR=/tmp/tmp.bzf4FYFzD4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jLmsCaaCoP ++ cat /tmp/tmp.bzf4FYFzD4 ++ rm /tmp/tmp.jLmsCaaCoP /tmp/tmp.bzf4FYFzD4 ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.bwlCpSWzTZ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.hrRbCRReGT +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.bwlCpSWzTZ +++++ cat /tmp/tmp.hrRbCRReGT +++++ rm /tmp/tmp.bwlCpSWzTZ /tmp/tmp.hrRbCRReGT +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.5zvylJFpY8 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.X4BATpiGOd +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.5zvylJFpY8 +++++ cat /tmp/tmp.X4BATpiGOd +++++ rm /tmp/tmp.5zvylJFpY8 /tmp/tmp.X4BATpiGOd +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Q410TEVga1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.LJnDSifXFH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Q410TEVga1 ++ cat /tmp/tmp.LJnDSifXFH ++ rm /tmp/tmp.Q410TEVga1 /tmp/tmp.LJnDSifXFH ++ return 0 + proxy_ready=3 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 3 == \3 ]] + echo + return + desc 'write data to source cluster' + set +o xtrace ----------------------------------------------------------------------------------- write data to source cluster ----------------------------------------------------------------------------------- + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testSourceReplica (id int PRIMARY KEY);' '-h 34.118.239.56 -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testSourceReplica (id int PRIMARY KEY);' + local 'uri=-h 34.118.239.56 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.snXXIp51jB +++ mktemp ++ local LAST_ERR=/tmp/tmp.6rDks62U09 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.snXXIp51jB ++ cat /tmp/tmp.6rDks62U09 ++ rm /tmp/tmp.snXXIp51jB /tmp/tmp.6rDks62U09 ++ return 0 + client_pod=pxc-client-56fd5498cd-c947g + wait_pod pxc-client-56fd5498cd-c947g + local pod=pxc-client-56fd5498cd-c947g + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-c947g ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 02:51:06.305785 20427 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782960664502029000&timeoutSeconds=372&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.testSourceReplica (id) VALUES (100700)' '-h 34.118.239.56 -uroot -proot_password' + local 'command=INSERT myApp.testSourceReplica (id) VALUES (100700)' + local 'uri=-h 34.118.239.56 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iDb8yUviud +++ mktemp ++ local LAST_ERR=/tmp/tmp.hAPkqwjdaj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iDb8yUviud ++ cat /tmp/tmp.hAPkqwjdaj ++ rm /tmp/tmp.iDb8yUviud /tmp/tmp.hAPkqwjdaj ++ return 0 + client_pod=pxc-client-56fd5498cd-c947g + wait_pod pxc-client-56fd5498cd-c947g + local pod=pxc-client-56fd5498cd-c947g + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-56fd5498cd-c947g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 02:51:15.956393 21933 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782960673880433000&timeoutSeconds=396&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + yq '.metadata={"name": "cross-site-replica-ssl-internal"}' - + yq 'del(.metadata)' - + kubectl_bin get secrets cross-site-source-ssl-internal -o yaml ++ mktemp + local LAST_OUT=/tmp/tmp.GS1N8bnXxQ ++ mktemp + local LAST_ERR=/tmp/tmp.McaSnvOorq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get secrets cross-site-source-ssl-internal -o yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GS1N8bnXxQ + cat /tmp/tmp.McaSnvOorq + rm /tmp/tmp.GS1N8bnXxQ /tmp/tmp.McaSnvOorq + return 0 + desc 'take backup of source cluster' + set +o xtrace ----------------------------------------------------------------------------------- take backup of source cluster ----------------------------------------------------------------------------------- + run_backup cross-site-source backup-minio-source + local cluster=cross-site-source + local backup=backup-minio-source + local status=Succeeded + log 'run pxc-backup/backup-minio-source' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T02:51:24+0000]' run pxc-backup/backup-minio-source [2026-07-02T02:51:24+0000] run pxc-backup/backup-minio-source + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/backup-minio-source.yml ++ mktemp + local LAST_OUT=/tmp/tmp.rTn7GGcK4D ++ mktemp + local LAST_ERR=/tmp/tmp.TxFX9PWBEp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/backup-minio-source.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rTn7GGcK4D perconaxtradbclusterbackup.pxc.percona.com/backup-minio-source created + cat /tmp/tmp.TxFX9PWBEp + rm /tmp/tmp.rTn7GGcK4D /tmp/tmp.TxFX9PWBEp + return 0 + wait_backup backup-minio-source Succeeded + local backup=backup-minio-source + local status=Succeeded + set +o xtrace waiting for pxc-backup/backup-minio-source to reach Succeeded state...............Succeeded + desc 'create replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- create replica cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-proxysql-replica-17697 0 + local namespace=cross-site-proxysql-replica-17697 + local skip_clean_namespace=0 + [[ 1 == 1 ]] + [[ -z 0 ]] + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-proxysql-replica-17697' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-proxysql-replica-17697 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-proxysql-replica-17697 ++ mktemp + local LAST_OUT=/tmp/tmp.2OtIMKivQu ++ mktemp + local LAST_ERR=/tmp/tmp.XIiYtxdUzL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-proxysql-replica-17697 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-proxysql-replica-17697 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-proxysql-replica-17697 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.2OtIMKivQu + cat /tmp/tmp.XIiYtxdUzL Error from server (NotFound): namespaces "cross-site-proxysql-replica-17697" not found + rm /tmp/tmp.2OtIMKivQu /tmp/tmp.XIiYtxdUzL + return 1 + : + wait_for_delete namespace/cross-site-proxysql-replica-17697 + local res=namespace/cross-site-proxysql-replica-17697 + echo -n 'waiting for namespace/cross-site-proxysql-replica-17697 to be deleted' waiting for namespace/cross-site-proxysql-replica-17697 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "cross-site-proxysql-replica-17697" not found + desc 'create namespace cross-site-proxysql-replica-17697' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-proxysql-replica-17697 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-proxysql-replica-17697 ++ mktemp + local LAST_OUT=/tmp/tmp.sR96AEjmRm ++ mktemp + local LAST_ERR=/tmp/tmp.YDQFaiqpyq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cross-site-proxysql-replica-17697 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sR96AEjmRm namespace/cross-site-proxysql-replica-17697 created + cat /tmp/tmp.YDQFaiqpyq + rm /tmp/tmp.sR96AEjmRm /tmp/tmp.YDQFaiqpyq + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.xdzAVB0Op2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.IjCYnsVtgf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xdzAVB0Op2 ++ cat /tmp/tmp.IjCYnsVtgf ++ rm /tmp/tmp.xdzAVB0Op2 /tmp/tmp.IjCYnsVtgf ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-replica-17697 ++ mktemp + local LAST_OUT=/tmp/tmp.iBqWc6XHTs ++ mktemp + local LAST_ERR=/tmp/tmp.mqliFZnEmJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-replica-17697 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iBqWc6XHTs Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7" modified. + cat /tmp/tmp.mqliFZnEmJ + rm /tmp/tmp.iBqWc6XHTs /tmp/tmp.mqliFZnEmJ + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.cBuedpMvwH ++ mktemp + local LAST_ERR=/tmp/tmp.CrRmjm77jm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cBuedpMvwH customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.CrRmjm77jm + rm /tmp/tmp.cBuedpMvwH /tmp/tmp.CrRmjm77jm + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.mhfRTBJgCg ++ mktemp + local LAST_ERR=/tmp/tmp.gfw8X6VBZq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mhfRTBJgCg customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.gfw8X6VBZq E0702 02:52:09.915310 29456 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusters.pxc.percona.com&resourceVersion=1782960728797971000&timeoutSeconds=505&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.mhfRTBJgCg /tmp/tmp.gfw8X6VBZq + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.LWLToiFbtS ++ mktemp + local LAST_ERR=/tmp/tmp.fM079ZuzaR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LWLToiFbtS customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.fM079ZuzaR E0702 02:52:11.937384 29670 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterbackups.pxc.percona.com&resourceVersion=1782960728797971000&timeoutSeconds=418&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.LWLToiFbtS /tmp/tmp.fM079ZuzaR + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.sTv9GhjGlL ++ mktemp + local LAST_ERR=/tmp/tmp.6QL4WDLsin + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sTv9GhjGlL customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.6QL4WDLsin E0702 02:52:14.222660 29976 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterrestores.pxc.percona.com&resourceVersion=1782960733799405000&timeoutSeconds=505&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.sTv9GhjGlL /tmp/tmp.6QL4WDLsin + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.eGMl0DGiGB ++ mktemp + local LAST_ERR=/tmp/tmp.ZzSBNaJyV0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eGMl0DGiGB clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.ZzSBNaJyV0 + rm /tmp/tmp.eGMl0DGiGB /tmp/tmp.ZzSBNaJyV0 + return 0 + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-operator.yaml + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - ++ mktemp + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a^' + local LAST_OUT=/tmp/tmp.SaQI8A7ADz ++ mktemp + local LAST_ERR=/tmp/tmp.mW3989Peki + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SaQI8A7ADz deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.mW3989Peki + rm /tmp/tmp.SaQI8A7ADz /tmp/tmp.mW3989Peki + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.unEFveLbN9 ++ mktemp + local LAST_ERR=/tmp/tmp.RB68Mou8Sk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.unEFveLbN9 pod/percona-xtradb-cluster-operator-65b5d85575-xzrbt condition met + cat /tmp/tmp.RB68Mou8Sk + rm /tmp/tmp.unEFveLbN9 /tmp/tmp.RB68Mou8Sk + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ local LAST_OUT=/tmp/tmp.rxsPqzZz5I +++ mktemp ++ local LAST_ERR=/tmp/tmp.YX1mnDwLIC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rxsPqzZz5I ++ cat /tmp/tmp.YX1mnDwLIC ++ rm /tmp/tmp.rxsPqzZz5I /tmp/tmp.YX1mnDwLIC ++ return 0 + wait_pod percona-xtradb-cluster-operator-65b5d85575-9dbzk 480 pxc-operator + local pod=percona-xtradb-cluster-operator-65b5d85575-9dbzk + local max_retry=480 + local ns=pxc-operator ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo percona-xtradb-cluster-operator-65b5d85575-9dbzk + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-65b5d85575-9dbzk condition met E0702 02:53:26.121311 5314 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-9dbzk&resourceVersion=1782960804170473000&timeoutSeconds=334&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-65b5d85575-9dbzk to become Ready.Ok + sleep 3 + kubectl_bin apply -f /tmp/tmp.xyHcWb7plX/replica-ssl-internal.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.6vRggP7tAU ++ mktemp + local LAST_ERR=/tmp/tmp.Et97RmoiEI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /tmp/tmp.xyHcWb7plX/replica-ssl-internal.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6vRggP7tAU secret/cross-site-replica-ssl-internal created + cat /tmp/tmp.Et97RmoiEI + rm /tmp/tmp.6vRggP7tAU /tmp/tmp.Et97RmoiEI + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.157snlZ4t5 ++ mktemp + local LAST_ERR=/tmp/tmp.kq9FBw0UgS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.157snlZ4t5 secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.kq9FBw0UgS + rm /tmp/tmp.157snlZ4t5 /tmp/tmp.kq9FBw0UgS + return 0 + spinup_pxc cross-site-replica /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-replica.yml + local cluster=cross-site-replica + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-replica.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.N9TuumLjGK ++ mktemp + local LAST_ERR=/tmp/tmp.yRy7WzWD8g + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.N9TuumLjGK secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.yRy7WzWD8g + rm /tmp/tmp.N9TuumLjGK /tmp/tmp.yRy7WzWD8g + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + kubectl_bin apply -f - + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.2IMnFjihD8 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-proxysql-28035~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_ERR=/tmp/tmp.TOJl0VByY5 + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2IMnFjihD8 deployment.apps/pxc-client created + cat /tmp/tmp.TOJl0VByY5 + rm /tmp/tmp.2IMnFjihD8 /tmp/tmp.TOJl0VByY5 + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-replica.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-replica.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-replica.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-replica.yml + local pvc_name= + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/cross-site-replica.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-proxysql-28035~ + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_OUT=/tmp/tmp.fqjY72vITA + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4-backup#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.4#' + local LAST_ERR=/tmp/tmp.MvOEPvMTvX + local exit_status=0 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fqjY72vITA perconaxtradbcluster.pxc.percona.com/cross-site-replica created + cat /tmp/tmp.MvOEPvMTvX + rm /tmp/tmp.fqjY72vITA /tmp/tmp.MvOEPvMTvX + return 0 + desc 'waiting pods to be running for cross-site-replica' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for cross-site-replica ----------------------------------------------------------------------------------- ++ get_proxy cross-site-replica ++ local target_cluster=cross-site-replica +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uT1RwMI9VH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qVLKF9DDZO +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.uT1RwMI9VH +++ cat /tmp/tmp.qVLKF9DDZO +++ rm /tmp/tmp.uT1RwMI9VH /tmp/tmp.qVLKF9DDZO +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zoTWwrRDqF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GPdEvbJF3d +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zoTWwrRDqF +++ cat /tmp/tmp.GPdEvbJF3d +++ rm /tmp/tmp.zoTWwrRDqF /tmp/tmp.GPdEvbJF3d +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo cross-site-replica-proxysql ++ return + local proxy=cross-site-replica-proxysql ++ get_proxy_size cross-site-replica ++ local cluster=cross-site-replica ++ local size=0 +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.S4d4jexFHo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.n0y3LIWvet +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.S4d4jexFHo +++ cat /tmp/tmp.n0y3LIWvet +++ rm /tmp/tmp.S4d4jexFHo /tmp/tmp.n0y3LIWvet +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.otSgQzQFNW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sscnkpbdVJ +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.otSgQzQFNW +++ cat /tmp/tmp.sscnkpbdVJ +++ rm /tmp/tmp.otSgQzQFNW /tmp/tmp.sscnkpbdVJ +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5T1GERyprG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.i5yCM00KKB +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.5T1GERyprG +++ cat /tmp/tmp.i5yCM00KKB +++ rm /tmp/tmp.5T1GERyprG /tmp/tmp.i5yCM00KKB +++ return 0 ++ size=3 ++ echo 3 + local proxy_size=3 + wait_for_running cross-site-replica-pxc 3 + local name=cross-site-replica-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z cross-site-replica-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for cross-site-replica-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for cross-site-replica-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-0 480 + local pod=cross-site-replica-pxc-0 + local max_retry=480 + local ns= ++ echo cross-site-replica-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/cross-site-replica-pxc-0 condition met waiting for pod/cross-site-replica-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-1 480 + local pod=cross-site-replica-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo cross-site-replica-pxc-1 + local container=pxc + set +o xtrace pod/cross-site-replica-pxc-1 condition met waiting for pod/cross-site-replica-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-2 480 + local pod=cross-site-replica-pxc-2 + local max_retry=480 + local ns= ++ echo cross-site-replica-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-replica-pxc-2 condition met waiting for pod/cross-site-replica-pxc-2 to become Ready.Ok + wait_for_running cross-site-replica-proxysql 3 + local name=cross-site-replica-proxysql + local size=3 + local last_pod + local max_retry=480 + [[ -z cross-site-replica-proxysql ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for cross-site-replica-proxysql' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for cross-site-replica-proxysql ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-proxysql-0 480 + local pod=cross-site-replica-proxysql-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo cross-site-replica-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace pod/cross-site-replica-proxysql-0 condition met E0702 02:57:50.406149 11873 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcross-site-replica-proxysql-0&resourceVersion=1782961067655749000&timeoutSeconds=580&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/cross-site-replica-proxysql-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-proxysql-1 480 + local pod=cross-site-replica-proxysql-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo cross-site-replica-proxysql-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace pod/cross-site-replica-proxysql-1 condition met E0702 02:58:02.576720 13576 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcross-site-replica-proxysql-1&resourceVersion=1782961077655865000&timeoutSeconds=360&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/cross-site-replica-proxysql-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-proxysql-2 480 + local pod=cross-site-replica-proxysql-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo cross-site-replica-proxysql-2 + local container=proxysql + set +o xtrace pod/cross-site-replica-proxysql-2 condition met E0702 02:58:17.065634 15767 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcross-site-replica-proxysql-2&resourceVersion=1782961096752431009&timeoutSeconds=457&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/cross-site-replica-proxysql-2 to become Ready.Ok + wait_cluster_consistency cross-site-replica 3 3 + local cluster_name=cross-site-replica + local pxc_size=3 + local proxy_size=3 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/cross-site-replica to be ready' waiting for pxc/cross-site-replica to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qEgfrbY8Ok +++ mktemp ++ local LAST_ERR=/tmp/tmp.98mvonskUC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qEgfrbY8Ok ++ cat /tmp/tmp.98mvonskUC ++ rm /tmp/tmp.qEgfrbY8Ok /tmp/tmp.98mvonskUC ++ return 0 + state=ready ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DvfEbdzpQ6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mv2oVDUkhQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DvfEbdzpQ6 ++ cat /tmp/tmp.mv2oVDUkhQ ++ rm /tmp/tmp.DvfEbdzpQ6 /tmp/tmp.mv2oVDUkhQ ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.HSAgcUngU6 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.TpzVQMbVUF +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.HSAgcUngU6 +++++ cat /tmp/tmp.TpzVQMbVUF +++++ rm /tmp/tmp.HSAgcUngU6 /tmp/tmp.TpzVQMbVUF +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.PviYZrYuQ2 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.bMc2CcoCUA +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.PviYZrYuQ2 +++++ cat /tmp/tmp.bMc2CcoCUA +++++ rm /tmp/tmp.PviYZrYuQ2 /tmp/tmp.bMc2CcoCUA +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Tp8oKNC81l +++ mktemp ++ local LAST_ERR=/tmp/tmp.VHoGq0kWCT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Tp8oKNC81l ++ cat /tmp/tmp.VHoGq0kWCT ++ rm /tmp/tmp.Tp8oKNC81l /tmp/tmp.VHoGq0kWCT ++ return 0 + proxy_ready=3 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 3 == \3 ]] + echo + return + sleep 10 ++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.EqAp5w5cIz +++ mktemp ++ local LAST_ERR=/tmp/tmp.K6srQ7UMHk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EqAp5w5cIz ++ cat /tmp/tmp.K6srQ7UMHk ++ rm /tmp/tmp.EqAp5w5cIz /tmp/tmp.K6srQ7UMHk ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.S9OCQ0jsbi +++ mktemp ++ local LAST_ERR=/tmp/tmp.eE0eSgQVuz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.S9OCQ0jsbi ++ cat /tmp/tmp.eE0eSgQVuz ++ rm /tmp/tmp.S9OCQ0jsbi /tmp/tmp.eE0eSgQVuz ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-56fd5498cd-f2ffc + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 02:59:00.802352 22483 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782961137936409000&timeoutSeconds=511&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data cross-site-replica 100500 no-proxy -P3306 + local cluster=cross-site-replica + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password cross-site-replica root ++ local cluster=cross-site-replica ++ local user=root +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HWdJ7EyYhn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WuJnoI947h +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.HWdJ7EyYhn +++ cat /tmp/tmp.WuJnoI947h +++ rm /tmp/tmp.HWdJ7EyYhn /tmp/tmp.WuJnoI947h +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aRu9gBS6pY +++ mktemp ++ local LAST_ERR=/tmp/tmp.bgeTUxstmY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aRu9gBS6pY ++ cat /tmp/tmp.bgeTUxstmY ++ rm /tmp/tmp.aRu9gBS6pY /tmp/tmp.bgeTUxstmY ++ return 0 + root_pass=root_password ++ get_proxy cross-site-replica ++ local target_cluster=cross-site-replica +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.wag0QpY01w ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wZ7KZLUvuM +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.wag0QpY01w +++ cat /tmp/tmp.wZ7KZLUvuM +++ rm /tmp/tmp.wag0QpY01w /tmp/tmp.wZ7KZLUvuM +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jGhzOvZOIY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DDqhzYwUZK +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.jGhzOvZOIY +++ cat /tmp/tmp.DDqhzYwUZK +++ rm /tmp/tmp.jGhzOvZOIY /tmp/tmp.DDqhzYwUZK +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo cross-site-replica-proxysql ++ return + proxy=cross-site-replica-proxysql + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h cross-site-replica-pxc-0.cross-site-replica-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h cross-site-replica-pxc-0.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h cross-site-replica-pxc-0.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6ymiIgYbsL +++ mktemp ++ local LAST_ERR=/tmp/tmp.MTMFKFk3XT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6ymiIgYbsL ++ cat /tmp/tmp.MTMFKFk3XT ++ rm /tmp/tmp.6ymiIgYbsL /tmp/tmp.MTMFKFk3XT ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-56fd5498cd-f2ffc + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 02:59:24.575626 25990 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782961162077134000&timeoutSeconds=441&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data cross-site-replica select-1 no-proxy -P3306 + local cluster=cross-site-replica + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password cross-site-replica root ++ local cluster=cross-site-replica ++ local user=root +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BmInuDRILp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.v2TSBNAx1Y +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.BmInuDRILp +++ cat /tmp/tmp.v2TSBNAx1Y +++ rm /tmp/tmp.BmInuDRILp /tmp/tmp.v2TSBNAx1Y +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.hQnzhgCz86 +++ mktemp ++ local LAST_ERR=/tmp/tmp.lerX8JO8Dk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hQnzhgCz86 ++ cat /tmp/tmp.lerX8JO8Dk ++ rm /tmp/tmp.hQnzhgCz86 /tmp/tmp.lerX8JO8Dk ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size cross-site-replica ++ local cluster=cross-site-replica ++ local size=0 +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Lq6Pob33x3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0ffd8HAuJx +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Lq6Pob33x3 +++ cat /tmp/tmp.0ffd8HAuJx +++ rm /tmp/tmp.Lq6Pob33x3 /tmp/tmp.0ffd8HAuJx +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h cross-site-replica-pxc-0.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h cross-site-replica-pxc-0.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h cross-site-replica-pxc-0.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h cross-site-replica-pxc-0.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nW635G9rPl +++ mktemp ++ local LAST_ERR=/tmp/tmp.i0NBQxENYH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nW635G9rPl ++ cat /tmp/tmp.i0NBQxENYH ++ rm /tmp/tmp.nW635G9rPl /tmp/tmp.i0NBQxENYH ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-f2ffc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:00:13.546224 860 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782961212662563000&timeoutSeconds=365&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1.sql /tmp/tmp.xyHcWb7plX/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h cross-site-replica-pxc-1.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h cross-site-replica-pxc-1.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h cross-site-replica-pxc-1.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h cross-site-replica-pxc-1.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cILAGPowbp +++ mktemp ++ local LAST_ERR=/tmp/tmp.JnJDZNr164 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cILAGPowbp ++ cat /tmp/tmp.JnJDZNr164 ++ rm /tmp/tmp.cILAGPowbp /tmp/tmp.JnJDZNr164 ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-56fd5498cd-f2ffc + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:00:28.175343 3072 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782961227173990000&timeoutSeconds=469&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1.sql /tmp/tmp.xyHcWb7plX/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h cross-site-replica-pxc-2.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h cross-site-replica-pxc-2.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h cross-site-replica-pxc-2.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h cross-site-replica-pxc-2.cross-site-replica-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AMlWwHXpbp +++ mktemp ++ local LAST_ERR=/tmp/tmp.4SwcpwvNkO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AMlWwHXpbp ++ cat /tmp/tmp.4SwcpwvNkO ++ rm /tmp/tmp.AMlWwHXpbp /tmp/tmp.4SwcpwvNkO ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-f2ffc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:00:44.657960 5218 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782961241213344000&timeoutSeconds=322&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-1.sql /tmp/tmp.xyHcWb7plX/select-1.sql + is_keyring_plugin_in_use cross-site-replica + local cluster=cross-site-replica + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + kubectl exec cross-site-replica-pxc-0 -c pxc -- ls /var/lib/mysql/mysqld.my + return 1 + wait_pxc_ready cross-site-replica + local cluster=cross-site-replica + local retry=0 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' + '[' ready = ready ']' + desc 'restore backup from source cluster' + set +o xtrace ----------------------------------------------------------------------------------- restore backup from source cluster ----------------------------------------------------------------------------------- ++ kubectl_bin get -n cross-site-proxysql-28035 pxc-backup backup-minio-source -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MQcEmWpY5y +++ mktemp ++ local LAST_ERR=/tmp/tmp.Mxvdf3y1hj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get -n cross-site-proxysql-28035 pxc-backup backup-minio-source -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MQcEmWpY5y ++ cat /tmp/tmp.Mxvdf3y1hj ++ rm /tmp/tmp.MQcEmWpY5y /tmp/tmp.Mxvdf3y1hj ++ return 0 + destination=s3://operator-testing/cross-site-source-2026-07-02-02:51:26-full + kubectl_bin apply -f - + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-proxysql-28035~ + /usr/bin/sed -e s~#cluster~cross-site-replica~ + /usr/bin/sed -e s~#destination~s3://operator-testing/cross-site-source-2026-07-02-02:51:26-full~ + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/restore-backup-minio.yml ++ mktemp + local LAST_OUT=/tmp/tmp.y5Eii8N5hH ++ mktemp + local LAST_ERR=/tmp/tmp.XEYGA1k8fW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.y5Eii8N5hH perconaxtradbclusterrestore.pxc.percona.com/backup-minio created + cat /tmp/tmp.XEYGA1k8fW + rm /tmp/tmp.y5Eii8N5hH /tmp/tmp.XEYGA1k8fW + return 0 + wait_cluster_consistency cross-site-replica 3 3 + local cluster_name=cross-site-replica + local pxc_size=3 + local proxy_size=3 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/cross-site-replica to be ready' waiting for pxc/cross-site-replica to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w4P9WzqbmK +++ mktemp ++ local LAST_ERR=/tmp/tmp.enBQsmfwIe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w4P9WzqbmK ++ cat /tmp/tmp.enBQsmfwIe ++ rm /tmp/tmp.w4P9WzqbmK /tmp/tmp.enBQsmfwIe ++ return 0 + state=stopping ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cTI0Nd6dIj +++ mktemp ++ local LAST_ERR=/tmp/tmp.HXZYN03zKb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cTI0Nd6dIj ++ cat /tmp/tmp.HXZYN03zKb ++ rm /tmp/tmp.cTI0Nd6dIj /tmp/tmp.HXZYN03zKb ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.UdUr7ZHkB7 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ayeEgIhgDl +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.UdUr7ZHkB7 +++++ cat /tmp/tmp.ayeEgIhgDl +++++ rm /tmp/tmp.UdUr7ZHkB7 /tmp/tmp.ayeEgIhgDl +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.VBM1fwyYN2 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.jzwrPLXpbA +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.VBM1fwyYN2 +++++ cat /tmp/tmp.jzwrPLXpbA +++++ rm /tmp/tmp.VBM1fwyYN2 /tmp/tmp.jzwrPLXpbA +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fpZH1kdPOu +++ mktemp ++ local LAST_ERR=/tmp/tmp.ISxapHPP1B ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fpZH1kdPOu ++ cat /tmp/tmp.ISxapHPP1B ++ rm /tmp/tmp.fpZH1kdPOu /tmp/tmp.ISxapHPP1B ++ return 0 + proxy_ready= + [[ stopping == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AwBgPoxTz4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.5G7IE0SBrY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AwBgPoxTz4 ++ cat /tmp/tmp.5G7IE0SBrY ++ rm /tmp/tmp.AwBgPoxTz4 /tmp/tmp.5G7IE0SBrY ++ return 0 + state=stopping ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AjrHj99iZn +++ mktemp ++ local LAST_ERR=/tmp/tmp.e2M9kiMRhl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AjrHj99iZn ++ cat /tmp/tmp.e2M9kiMRhl ++ rm /tmp/tmp.AjrHj99iZn /tmp/tmp.e2M9kiMRhl ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.oWgP4vLu1T ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.7fzhwbqq5E +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.oWgP4vLu1T +++++ cat /tmp/tmp.7fzhwbqq5E +++++ rm /tmp/tmp.oWgP4vLu1T /tmp/tmp.7fzhwbqq5E +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.pCb8zUvPj6 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.TxDFmgjQJs +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.pCb8zUvPj6 +++++ cat /tmp/tmp.TxDFmgjQJs +++++ rm /tmp/tmp.pCb8zUvPj6 /tmp/tmp.TxDFmgjQJs +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.08bscBxdTd +++ mktemp ++ local LAST_ERR=/tmp/tmp.GjVIeXWtH9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.08bscBxdTd ++ cat /tmp/tmp.GjVIeXWtH9 ++ rm /tmp/tmp.08bscBxdTd /tmp/tmp.GjVIeXWtH9 ++ return 0 + proxy_ready= + [[ stopping == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NCKSoNqSqV +++ mktemp ++ local LAST_ERR=/tmp/tmp.7UZNJ83eYd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NCKSoNqSqV ++ cat /tmp/tmp.7UZNJ83eYd ++ rm /tmp/tmp.NCKSoNqSqV /tmp/tmp.7UZNJ83eYd ++ return 0 + state=stopping ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OIzK3ARefU +++ mktemp ++ local LAST_ERR=/tmp/tmp.TNNleCTn9s ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OIzK3ARefU ++ cat /tmp/tmp.TNNleCTn9s ++ rm /tmp/tmp.OIzK3ARefU /tmp/tmp.TNNleCTn9s ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.yF2sC06MBA ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.NYQ5jSfaGw +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.yF2sC06MBA +++++ cat /tmp/tmp.NYQ5jSfaGw +++++ rm /tmp/tmp.yF2sC06MBA /tmp/tmp.NYQ5jSfaGw +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.kGmBoJ2L2r ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.o2LnX7N6PD +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.kGmBoJ2L2r +++++ cat /tmp/tmp.o2LnX7N6PD +++++ rm /tmp/tmp.kGmBoJ2L2r /tmp/tmp.o2LnX7N6PD +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0RCjBGYhAs +++ mktemp ++ local LAST_ERR=/tmp/tmp.2ssNTBalhO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0RCjBGYhAs ++ cat /tmp/tmp.2ssNTBalhO ++ rm /tmp/tmp.0RCjBGYhAs /tmp/tmp.2ssNTBalhO ++ return 0 + proxy_ready= + [[ stopping == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AEIkAABX12 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Q9ihX12lwl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AEIkAABX12 ++ cat /tmp/tmp.Q9ihX12lwl ++ rm /tmp/tmp.AEIkAABX12 /tmp/tmp.Q9ihX12lwl ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.b4Tgq65cPq +++ mktemp ++ local LAST_ERR=/tmp/tmp.JKgdA58VYA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.b4Tgq65cPq ++ cat /tmp/tmp.JKgdA58VYA ++ rm /tmp/tmp.b4Tgq65cPq /tmp/tmp.JKgdA58VYA ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.92fB48KPmv ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.7KwAHirLJG +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.92fB48KPmv +++++ cat /tmp/tmp.7KwAHirLJG +++++ rm /tmp/tmp.92fB48KPmv /tmp/tmp.7KwAHirLJG +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.QIVVtdGB4P ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.vEoTJWb2HM +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.QIVVtdGB4P +++++ cat /tmp/tmp.vEoTJWb2HM +++++ rm /tmp/tmp.QIVVtdGB4P /tmp/tmp.vEoTJWb2HM +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vJx7r9sgsp +++ mktemp ++ local LAST_ERR=/tmp/tmp.PDDGfB8HbU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vJx7r9sgsp ++ cat /tmp/tmp.PDDGfB8HbU ++ rm /tmp/tmp.vJx7r9sgsp /tmp/tmp.PDDGfB8HbU ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HvbqzZxF0t +++ mktemp ++ local LAST_ERR=/tmp/tmp.snhqsxDiLh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HvbqzZxF0t ++ cat /tmp/tmp.snhqsxDiLh ++ rm /tmp/tmp.HvbqzZxF0t /tmp/tmp.snhqsxDiLh ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0FXHbcf611 +++ mktemp ++ local LAST_ERR=/tmp/tmp.P7WWiSct6r ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0FXHbcf611 ++ cat /tmp/tmp.P7WWiSct6r ++ rm /tmp/tmp.0FXHbcf611 /tmp/tmp.P7WWiSct6r ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.eVabJ5JQ2I ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.tLQWZVyiQd +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.eVabJ5JQ2I +++++ cat /tmp/tmp.tLQWZVyiQd +++++ rm /tmp/tmp.eVabJ5JQ2I /tmp/tmp.tLQWZVyiQd +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.1igB6eqcAL ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ezn7H649PC +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.1igB6eqcAL +++++ cat /tmp/tmp.ezn7H649PC +++++ rm /tmp/tmp.1igB6eqcAL /tmp/tmp.ezn7H649PC +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Kp1yJfqD6V +++ mktemp ++ local LAST_ERR=/tmp/tmp.B2TXbqKbS2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Kp1yJfqD6V ++ cat /tmp/tmp.B2TXbqKbS2 ++ rm /tmp/tmp.Kp1yJfqD6V /tmp/tmp.B2TXbqKbS2 ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QAlmtt8bI6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.XaKEpUM1Bb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QAlmtt8bI6 ++ cat /tmp/tmp.XaKEpUM1Bb ++ rm /tmp/tmp.QAlmtt8bI6 /tmp/tmp.XaKEpUM1Bb ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aZ6TUCLjFe +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ay2anoP249 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aZ6TUCLjFe ++ cat /tmp/tmp.Ay2anoP249 ++ rm /tmp/tmp.aZ6TUCLjFe /tmp/tmp.Ay2anoP249 ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.JNr1Pd14cA ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.wQk3HmSWLL +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.JNr1Pd14cA +++++ cat /tmp/tmp.wQk3HmSWLL +++++ rm /tmp/tmp.JNr1Pd14cA /tmp/tmp.wQk3HmSWLL +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.9w3JA6Mzt9 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.5vuLdTD1HJ +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.9w3JA6Mzt9 +++++ cat /tmp/tmp.5vuLdTD1HJ +++++ rm /tmp/tmp.9w3JA6Mzt9 /tmp/tmp.5vuLdTD1HJ +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eJYqTRlGtT +++ mktemp ++ local LAST_ERR=/tmp/tmp.PowSW7nePS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eJYqTRlGtT ++ cat /tmp/tmp.PowSW7nePS ++ rm /tmp/tmp.eJYqTRlGtT /tmp/tmp.PowSW7nePS ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9VutFP6BqG +++ mktemp ++ local LAST_ERR=/tmp/tmp.r2j4M1rWzZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9VutFP6BqG ++ cat /tmp/tmp.r2j4M1rWzZ ++ rm /tmp/tmp.9VutFP6BqG /tmp/tmp.r2j4M1rWzZ ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BzbKP4xVLi +++ mktemp ++ local LAST_ERR=/tmp/tmp.YN4veh9gDo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BzbKP4xVLi ++ cat /tmp/tmp.YN4veh9gDo ++ rm /tmp/tmp.BzbKP4xVLi /tmp/tmp.YN4veh9gDo ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.tdLgeyGxIH ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ld1OFDmEni +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.tdLgeyGxIH +++++ cat /tmp/tmp.ld1OFDmEni +++++ rm /tmp/tmp.tdLgeyGxIH /tmp/tmp.ld1OFDmEni +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.SYJ0TkGRXP ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.HzM6jdMVY1 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.SYJ0TkGRXP +++++ cat /tmp/tmp.HzM6jdMVY1 +++++ rm /tmp/tmp.SYJ0TkGRXP /tmp/tmp.HzM6jdMVY1 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PIbkb9bycP +++ mktemp ++ local LAST_ERR=/tmp/tmp.Q8Lphb9Po4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PIbkb9bycP ++ cat /tmp/tmp.Q8Lphb9Po4 ++ rm /tmp/tmp.PIbkb9bycP /tmp/tmp.Q8Lphb9Po4 ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Np6mu5pQL5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.xPmuHWNMrJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Np6mu5pQL5 ++ cat /tmp/tmp.xPmuHWNMrJ ++ rm /tmp/tmp.Np6mu5pQL5 /tmp/tmp.xPmuHWNMrJ ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lNOdLFwtjq +++ mktemp ++ local LAST_ERR=/tmp/tmp.JGaMYQDnQE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lNOdLFwtjq ++ cat /tmp/tmp.JGaMYQDnQE ++ rm /tmp/tmp.lNOdLFwtjq /tmp/tmp.JGaMYQDnQE ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.HUoZ60dsND ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.BeAOxXquZL +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.HUoZ60dsND +++++ cat /tmp/tmp.BeAOxXquZL +++++ rm /tmp/tmp.HUoZ60dsND /tmp/tmp.BeAOxXquZL +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.hBm8YQIlKh ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.0vVi7dGF49 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.hBm8YQIlKh +++++ cat /tmp/tmp.0vVi7dGF49 +++++ rm /tmp/tmp.hBm8YQIlKh /tmp/tmp.0vVi7dGF49 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pdvWprhaMC +++ mktemp ++ local LAST_ERR=/tmp/tmp.NNh1CYB9om ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pdvWprhaMC ++ cat /tmp/tmp.NNh1CYB9om ++ rm /tmp/tmp.pdvWprhaMC /tmp/tmp.NNh1CYB9om ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.smpaQa4gSs +++ mktemp ++ local LAST_ERR=/tmp/tmp.Mz1oHyYHXH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.smpaQa4gSs ++ cat /tmp/tmp.Mz1oHyYHXH ++ rm /tmp/tmp.smpaQa4gSs /tmp/tmp.Mz1oHyYHXH ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hwYndj0c9h +++ mktemp ++ local LAST_ERR=/tmp/tmp.eBChwSsULt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hwYndj0c9h ++ cat /tmp/tmp.eBChwSsULt ++ rm /tmp/tmp.hwYndj0c9h /tmp/tmp.eBChwSsULt ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.YR4I96cuNs ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.t7NeEM0rZj +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.YR4I96cuNs +++++ cat /tmp/tmp.t7NeEM0rZj +++++ rm /tmp/tmp.YR4I96cuNs /tmp/tmp.t7NeEM0rZj +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.PeKQbqey7C ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Y9vv1evZCb +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.PeKQbqey7C +++++ cat /tmp/tmp.Y9vv1evZCb +++++ rm /tmp/tmp.PeKQbqey7C /tmp/tmp.Y9vv1evZCb +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mfUEE1ZC0b +++ mktemp ++ local LAST_ERR=/tmp/tmp.qS3AjuvRTS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mfUEE1ZC0b ++ cat /tmp/tmp.qS3AjuvRTS ++ rm /tmp/tmp.mfUEE1ZC0b /tmp/tmp.qS3AjuvRTS ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AaIwnWP4Dt +++ mktemp ++ local LAST_ERR=/tmp/tmp.txf8aQYgyl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AaIwnWP4Dt ++ cat /tmp/tmp.txf8aQYgyl ++ rm /tmp/tmp.AaIwnWP4Dt /tmp/tmp.txf8aQYgyl ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sdjocpD14J +++ mktemp ++ local LAST_ERR=/tmp/tmp.9JgyhYaX1Y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sdjocpD14J ++ cat /tmp/tmp.9JgyhYaX1Y ++ rm /tmp/tmp.sdjocpD14J /tmp/tmp.9JgyhYaX1Y ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.sEU8CYsf9E ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.dtAg8yBvTV +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.sEU8CYsf9E +++++ cat /tmp/tmp.dtAg8yBvTV +++++ rm /tmp/tmp.sEU8CYsf9E /tmp/tmp.dtAg8yBvTV +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.yrui8CyDTg ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.4nHPu9CuaJ +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.yrui8CyDTg +++++ cat /tmp/tmp.4nHPu9CuaJ +++++ rm /tmp/tmp.yrui8CyDTg /tmp/tmp.4nHPu9CuaJ +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mMuDo9FD2i +++ mktemp ++ local LAST_ERR=/tmp/tmp.OKwxzaU28I ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mMuDo9FD2i ++ cat /tmp/tmp.OKwxzaU28I ++ rm /tmp/tmp.mMuDo9FD2i /tmp/tmp.OKwxzaU28I ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d2vZ9ol8Xn +++ mktemp ++ local LAST_ERR=/tmp/tmp.nq7RzLRCcm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.d2vZ9ol8Xn ++ cat /tmp/tmp.nq7RzLRCcm ++ rm /tmp/tmp.d2vZ9ol8Xn /tmp/tmp.nq7RzLRCcm ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BvMX0HyJH1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.9HBgNtjHv3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BvMX0HyJH1 ++ cat /tmp/tmp.9HBgNtjHv3 ++ rm /tmp/tmp.BvMX0HyJH1 /tmp/tmp.9HBgNtjHv3 ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.bJfkxVWy6g ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.SU9waGeBzO +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.bJfkxVWy6g +++++ cat /tmp/tmp.SU9waGeBzO +++++ rm /tmp/tmp.bJfkxVWy6g /tmp/tmp.SU9waGeBzO +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.2chCqQh1PG ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.j47Qo5Dd4y +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.2chCqQh1PG +++++ cat /tmp/tmp.j47Qo5Dd4y +++++ rm /tmp/tmp.2chCqQh1PG /tmp/tmp.j47Qo5Dd4y +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HsMCizZcsL +++ mktemp ++ local LAST_ERR=/tmp/tmp.D14VwLkMib ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HsMCizZcsL ++ cat /tmp/tmp.D14VwLkMib ++ rm /tmp/tmp.HsMCizZcsL /tmp/tmp.D14VwLkMib ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GJVYidZrer +++ mktemp ++ local LAST_ERR=/tmp/tmp.AbA6KslOWP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GJVYidZrer ++ cat /tmp/tmp.AbA6KslOWP ++ rm /tmp/tmp.GJVYidZrer /tmp/tmp.AbA6KslOWP ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CmMGQCBUc3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.XnBEhNNQ71 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CmMGQCBUc3 ++ cat /tmp/tmp.XnBEhNNQ71 ++ rm /tmp/tmp.CmMGQCBUc3 /tmp/tmp.XnBEhNNQ71 ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.5LcGC9dm0F ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.cvVY9jAtIm +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.5LcGC9dm0F +++++ cat /tmp/tmp.cvVY9jAtIm +++++ rm /tmp/tmp.5LcGC9dm0F /tmp/tmp.cvVY9jAtIm +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.1zuvmaN3n5 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.VouRQTzrPr +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.1zuvmaN3n5 +++++ cat /tmp/tmp.VouRQTzrPr +++++ rm /tmp/tmp.1zuvmaN3n5 /tmp/tmp.VouRQTzrPr +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zg9Gr24Dwo +++ mktemp ++ local LAST_ERR=/tmp/tmp.vroutu85FD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zg9Gr24Dwo ++ cat /tmp/tmp.vroutu85FD ++ rm /tmp/tmp.zg9Gr24Dwo /tmp/tmp.vroutu85FD ++ return 0 + proxy_ready=1 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xSvwVVBLOu +++ mktemp ++ local LAST_ERR=/tmp/tmp.wbwY0bVpOU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xSvwVVBLOu ++ cat /tmp/tmp.wbwY0bVpOU ++ rm /tmp/tmp.xSvwVVBLOu /tmp/tmp.wbwY0bVpOU ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n530KmHnds +++ mktemp ++ local LAST_ERR=/tmp/tmp.iNzmD4ug5x ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.n530KmHnds ++ cat /tmp/tmp.iNzmD4ug5x ++ rm /tmp/tmp.n530KmHnds /tmp/tmp.iNzmD4ug5x ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.zS4wotDd8V ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.fNjuSP9IDI +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.zS4wotDd8V +++++ cat /tmp/tmp.fNjuSP9IDI +++++ rm /tmp/tmp.zS4wotDd8V /tmp/tmp.fNjuSP9IDI +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.H2MWFdOXWY ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.CU9ovNbZzJ +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.H2MWFdOXWY +++++ cat /tmp/tmp.CU9ovNbZzJ +++++ rm /tmp/tmp.H2MWFdOXWY /tmp/tmp.CU9ovNbZzJ +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n6f703VIkD +++ mktemp ++ local LAST_ERR=/tmp/tmp.EOjBCcqFXR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.n6f703VIkD ++ cat /tmp/tmp.EOjBCcqFXR ++ rm /tmp/tmp.n6f703VIkD /tmp/tmp.EOjBCcqFXR ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3gyge4Smea +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZTGLMjBQqs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3gyge4Smea ++ cat /tmp/tmp.ZTGLMjBQqs ++ rm /tmp/tmp.3gyge4Smea /tmp/tmp.ZTGLMjBQqs ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sLSG8Cd0XI +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ugt0ykmrJb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sLSG8Cd0XI ++ cat /tmp/tmp.Ugt0ykmrJb ++ rm /tmp/tmp.sLSG8Cd0XI /tmp/tmp.Ugt0ykmrJb ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.r7EzIBQidn ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.wUGBgZsCRQ +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.r7EzIBQidn +++++ cat /tmp/tmp.wUGBgZsCRQ +++++ rm /tmp/tmp.r7EzIBQidn /tmp/tmp.wUGBgZsCRQ +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.YY53vCfutw ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.s7PcEwPDKO +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.YY53vCfutw +++++ cat /tmp/tmp.s7PcEwPDKO +++++ rm /tmp/tmp.YY53vCfutw /tmp/tmp.s7PcEwPDKO +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9VCh2A1lnh +++ mktemp ++ local LAST_ERR=/tmp/tmp.bCH4ez3CxE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9VCh2A1lnh ++ cat /tmp/tmp.bCH4ez3CxE ++ rm /tmp/tmp.9VCh2A1lnh /tmp/tmp.bCH4ez3CxE ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8ZpQBLGSk5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FSQ7u1kEPn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8ZpQBLGSk5 ++ cat /tmp/tmp.FSQ7u1kEPn ++ rm /tmp/tmp.8ZpQBLGSk5 /tmp/tmp.FSQ7u1kEPn ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xdD58xonla +++ mktemp ++ local LAST_ERR=/tmp/tmp.fVHXiiLBwT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xdD58xonla ++ cat /tmp/tmp.fVHXiiLBwT ++ rm /tmp/tmp.xdD58xonla /tmp/tmp.fVHXiiLBwT ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.oLLLei6OVW ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.AmKLE9a7YR +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.oLLLei6OVW +++++ cat /tmp/tmp.AmKLE9a7YR +++++ rm /tmp/tmp.oLLLei6OVW /tmp/tmp.AmKLE9a7YR +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Dlw1z8nEdK ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.tc7bhyaBGH +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Dlw1z8nEdK +++++ cat /tmp/tmp.tc7bhyaBGH +++++ rm /tmp/tmp.Dlw1z8nEdK /tmp/tmp.tc7bhyaBGH +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A4gcVgpysT +++ mktemp ++ local LAST_ERR=/tmp/tmp.qVAjaCW5Wo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A4gcVgpysT ++ cat /tmp/tmp.qVAjaCW5Wo ++ rm /tmp/tmp.A4gcVgpysT /tmp/tmp.qVAjaCW5Wo ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bsYqstdbF8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.szw2B9giiF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bsYqstdbF8 ++ cat /tmp/tmp.szw2B9giiF ++ rm /tmp/tmp.bsYqstdbF8 /tmp/tmp.szw2B9giiF ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mwC2WNHu0q +++ mktemp ++ local LAST_ERR=/tmp/tmp.bDfrma9Lf0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mwC2WNHu0q ++ cat /tmp/tmp.bDfrma9Lf0 ++ rm /tmp/tmp.mwC2WNHu0q /tmp/tmp.bDfrma9Lf0 ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.YHsJdRl8cb ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.xV5AqJl2JR +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.YHsJdRl8cb +++++ cat /tmp/tmp.xV5AqJl2JR +++++ rm /tmp/tmp.YHsJdRl8cb /tmp/tmp.xV5AqJl2JR +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.D9mU5OKgK7 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.oICmUxHY3E +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.D9mU5OKgK7 +++++ cat /tmp/tmp.oICmUxHY3E +++++ rm /tmp/tmp.D9mU5OKgK7 /tmp/tmp.oICmUxHY3E +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.buDrtulfmN +++ mktemp ++ local LAST_ERR=/tmp/tmp.xm54qXKzLS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.buDrtulfmN ++ cat /tmp/tmp.xm54qXKzLS ++ rm /tmp/tmp.buDrtulfmN /tmp/tmp.xm54qXKzLS ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3NT4mPktNG +++ mktemp ++ local LAST_ERR=/tmp/tmp.TrPd0vkVe6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3NT4mPktNG ++ cat /tmp/tmp.TrPd0vkVe6 ++ rm /tmp/tmp.3NT4mPktNG /tmp/tmp.TrPd0vkVe6 ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qg99qs4o0S +++ mktemp ++ local LAST_ERR=/tmp/tmp.aB4FBhfcXF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qg99qs4o0S ++ cat /tmp/tmp.aB4FBhfcXF ++ rm /tmp/tmp.qg99qs4o0S /tmp/tmp.aB4FBhfcXF ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.5ZhGhaS8hV ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Pdjfb5vvsd +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.5ZhGhaS8hV +++++ cat /tmp/tmp.Pdjfb5vvsd +++++ rm /tmp/tmp.5ZhGhaS8hV /tmp/tmp.Pdjfb5vvsd +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.CuBLm7Bu73 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.y2NFc5myXO +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.CuBLm7Bu73 +++++ cat /tmp/tmp.y2NFc5myXO +++++ rm /tmp/tmp.CuBLm7Bu73 /tmp/tmp.y2NFc5myXO +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6EJi7SpnDw +++ mktemp ++ local LAST_ERR=/tmp/tmp.bOaPaJgEp0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6EJi7SpnDw ++ cat /tmp/tmp.bOaPaJgEp0 ++ rm /tmp/tmp.6EJi7SpnDw /tmp/tmp.bOaPaJgEp0 ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.57KAlLqv1h +++ mktemp ++ local LAST_ERR=/tmp/tmp.kXeMtp3DpU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.57KAlLqv1h ++ cat /tmp/tmp.kXeMtp3DpU ++ rm /tmp/tmp.57KAlLqv1h /tmp/tmp.kXeMtp3DpU ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TYWUwi5dW1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.b4da0yznlo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TYWUwi5dW1 ++ cat /tmp/tmp.b4da0yznlo ++ rm /tmp/tmp.TYWUwi5dW1 /tmp/tmp.b4da0yznlo ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.uTXsdSrV6k ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.uWrhzDfa4W +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.uTXsdSrV6k +++++ cat /tmp/tmp.uWrhzDfa4W +++++ rm /tmp/tmp.uTXsdSrV6k /tmp/tmp.uWrhzDfa4W +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.yrFtvv6vtS ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.QY9AobVZxc +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.yrFtvv6vtS +++++ cat /tmp/tmp.QY9AobVZxc +++++ rm /tmp/tmp.yrFtvv6vtS /tmp/tmp.QY9AobVZxc +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.brzczO9guk +++ mktemp ++ local LAST_ERR=/tmp/tmp.5cD9QnZ6K7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.brzczO9guk ++ cat /tmp/tmp.5cD9QnZ6K7 ++ rm /tmp/tmp.brzczO9guk /tmp/tmp.5cD9QnZ6K7 ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ppJciBkUVb +++ mktemp ++ local LAST_ERR=/tmp/tmp.sHWF58fsmh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ppJciBkUVb ++ cat /tmp/tmp.sHWF58fsmh ++ rm /tmp/tmp.ppJciBkUVb /tmp/tmp.sHWF58fsmh ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RGkyMJe82o +++ mktemp ++ local LAST_ERR=/tmp/tmp.bhSQZKqRbk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RGkyMJe82o ++ cat /tmp/tmp.bhSQZKqRbk ++ rm /tmp/tmp.RGkyMJe82o /tmp/tmp.bhSQZKqRbk ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.fZSEtul5HS ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.N3wuQEhUf2 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.fZSEtul5HS +++++ cat /tmp/tmp.N3wuQEhUf2 +++++ rm /tmp/tmp.fZSEtul5HS /tmp/tmp.N3wuQEhUf2 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.3rKqh3gEXf ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.5vdXlS7jKJ +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.3rKqh3gEXf +++++ cat /tmp/tmp.5vdXlS7jKJ +++++ rm /tmp/tmp.3rKqh3gEXf /tmp/tmp.5vdXlS7jKJ +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6nHHATZ2oq +++ mktemp ++ local LAST_ERR=/tmp/tmp.1GpjU5Ml4S ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6nHHATZ2oq ++ cat /tmp/tmp.1GpjU5Ml4S ++ rm /tmp/tmp.6nHHATZ2oq /tmp/tmp.1GpjU5Ml4S ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZQJEsVQWHs +++ mktemp ++ local LAST_ERR=/tmp/tmp.5zcgoSkb2m ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZQJEsVQWHs ++ cat /tmp/tmp.5zcgoSkb2m ++ rm /tmp/tmp.ZQJEsVQWHs /tmp/tmp.5zcgoSkb2m ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.owKSUVyvLs +++ mktemp ++ local LAST_ERR=/tmp/tmp.eW41abKQVa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.owKSUVyvLs ++ cat /tmp/tmp.eW41abKQVa ++ rm /tmp/tmp.owKSUVyvLs /tmp/tmp.eW41abKQVa ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.PGZpnTrigd ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.DfjYxCE57K +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.PGZpnTrigd +++++ cat /tmp/tmp.DfjYxCE57K +++++ rm /tmp/tmp.PGZpnTrigd /tmp/tmp.DfjYxCE57K +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.va69k9resI ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.R57jlkL0Qh +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.va69k9resI +++++ cat /tmp/tmp.R57jlkL0Qh +++++ rm /tmp/tmp.va69k9resI /tmp/tmp.R57jlkL0Qh +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rb4KQlJ9je +++ mktemp ++ local LAST_ERR=/tmp/tmp.L4i9cIxoLh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rb4KQlJ9je ++ cat /tmp/tmp.L4i9cIxoLh ++ rm /tmp/tmp.rb4KQlJ9je /tmp/tmp.L4i9cIxoLh ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IZzykZSJHY +++ mktemp ++ local LAST_ERR=/tmp/tmp.LLDqRPKDCQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IZzykZSJHY ++ cat /tmp/tmp.LLDqRPKDCQ ++ rm /tmp/tmp.IZzykZSJHY /tmp/tmp.LLDqRPKDCQ ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8uZhp41uMt +++ mktemp ++ local LAST_ERR=/tmp/tmp.lQuk6xHGS6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8uZhp41uMt ++ cat /tmp/tmp.lQuk6xHGS6 ++ rm /tmp/tmp.8uZhp41uMt /tmp/tmp.lQuk6xHGS6 ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.8LWKnfMT5M ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.15IbtHDvUc +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.8LWKnfMT5M +++++ cat /tmp/tmp.15IbtHDvUc +++++ rm /tmp/tmp.8LWKnfMT5M /tmp/tmp.15IbtHDvUc +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.9T95EPLb5x ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.fciePRbdtk +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.9T95EPLb5x +++++ cat /tmp/tmp.fciePRbdtk +++++ rm /tmp/tmp.9T95EPLb5x /tmp/tmp.fciePRbdtk +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wFCtjpE5OB +++ mktemp ++ local LAST_ERR=/tmp/tmp.GjhOBzQab8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wFCtjpE5OB ++ cat /tmp/tmp.GjhOBzQab8 ++ rm /tmp/tmp.wFCtjpE5OB /tmp/tmp.GjhOBzQab8 ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pKPBBg8cEU +++ mktemp ++ local LAST_ERR=/tmp/tmp.71I5GR3L33 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pKPBBg8cEU ++ cat /tmp/tmp.71I5GR3L33 ++ rm /tmp/tmp.pKPBBg8cEU /tmp/tmp.71I5GR3L33 ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Utqnzd4CfJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.wvefuKereb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Utqnzd4CfJ ++ cat /tmp/tmp.wvefuKereb ++ rm /tmp/tmp.Utqnzd4CfJ /tmp/tmp.wvefuKereb ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Peh3UnJKQP ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.VX9ZVUAtXT +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Peh3UnJKQP +++++ cat /tmp/tmp.VX9ZVUAtXT +++++ rm /tmp/tmp.Peh3UnJKQP /tmp/tmp.VX9ZVUAtXT +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.pkdMy3SDoO ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.n23ZgMyKhz +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.pkdMy3SDoO +++++ cat /tmp/tmp.n23ZgMyKhz +++++ rm /tmp/tmp.pkdMy3SDoO /tmp/tmp.n23ZgMyKhz +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.epVpA6m1JD +++ mktemp ++ local LAST_ERR=/tmp/tmp.suOWLGsBDg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.epVpA6m1JD ++ cat /tmp/tmp.suOWLGsBDg ++ rm /tmp/tmp.epVpA6m1JD /tmp/tmp.suOWLGsBDg ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xhWloAX2pR +++ mktemp ++ local LAST_ERR=/tmp/tmp.6jLGG2B5N3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xhWloAX2pR ++ cat /tmp/tmp.6jLGG2B5N3 ++ rm /tmp/tmp.xhWloAX2pR /tmp/tmp.6jLGG2B5N3 ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gZ2EQ52g21 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Q1r5TTlepZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gZ2EQ52g21 ++ cat /tmp/tmp.Q1r5TTlepZ ++ rm /tmp/tmp.gZ2EQ52g21 /tmp/tmp.Q1r5TTlepZ ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.0Snamw21MZ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.02SY6vDC3I +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.0Snamw21MZ +++++ cat /tmp/tmp.02SY6vDC3I +++++ rm /tmp/tmp.0Snamw21MZ /tmp/tmp.02SY6vDC3I +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Ilw7hXbzys ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Uk0G1NVwS8 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Ilw7hXbzys +++++ cat /tmp/tmp.Uk0G1NVwS8 +++++ rm /tmp/tmp.Ilw7hXbzys /tmp/tmp.Uk0G1NVwS8 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eW8vkxtAF5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.e8uUGjSN2H ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eW8vkxtAF5 ++ cat /tmp/tmp.e8uUGjSN2H ++ rm /tmp/tmp.eW8vkxtAF5 /tmp/tmp.e8uUGjSN2H ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UjjTgiH8Jz +++ mktemp ++ local LAST_ERR=/tmp/tmp.4uBTkRK8ej ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UjjTgiH8Jz ++ cat /tmp/tmp.4uBTkRK8ej ++ rm /tmp/tmp.UjjTgiH8Jz /tmp/tmp.4uBTkRK8ej ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pI8LITb6kb +++ mktemp ++ local LAST_ERR=/tmp/tmp.PbqnLTCBfn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pI8LITb6kb ++ cat /tmp/tmp.PbqnLTCBfn ++ rm /tmp/tmp.pI8LITb6kb /tmp/tmp.PbqnLTCBfn ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.YhEd9J7P3v ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.qZmM0NIGyj +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.YhEd9J7P3v +++++ cat /tmp/tmp.qZmM0NIGyj +++++ rm /tmp/tmp.YhEd9J7P3v /tmp/tmp.qZmM0NIGyj +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.3614s1FKWH ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.QS1yd27gbi +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.3614s1FKWH +++++ cat /tmp/tmp.QS1yd27gbi +++++ rm /tmp/tmp.3614s1FKWH /tmp/tmp.QS1yd27gbi +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HvKJP4q3aa +++ mktemp ++ local LAST_ERR=/tmp/tmp.nfwbqEfeHK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HvKJP4q3aa ++ cat /tmp/tmp.nfwbqEfeHK ++ rm /tmp/tmp.HvKJP4q3aa /tmp/tmp.nfwbqEfeHK ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7a294AeCNA +++ mktemp ++ local LAST_ERR=/tmp/tmp.APNXMsp3o3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7a294AeCNA ++ cat /tmp/tmp.APNXMsp3o3 ++ rm /tmp/tmp.7a294AeCNA /tmp/tmp.APNXMsp3o3 ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nXlDfa9dIU +++ mktemp ++ local LAST_ERR=/tmp/tmp.UL6zaqrvQs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nXlDfa9dIU ++ cat /tmp/tmp.UL6zaqrvQs ++ rm /tmp/tmp.nXlDfa9dIU /tmp/tmp.UL6zaqrvQs ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.JaJfppHyvc ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.UavAWgOq3b +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.JaJfppHyvc +++++ cat /tmp/tmp.UavAWgOq3b +++++ rm /tmp/tmp.JaJfppHyvc /tmp/tmp.UavAWgOq3b +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.C2smP0DWWc ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.J9Ajcjeklc +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.C2smP0DWWc +++++ cat /tmp/tmp.J9Ajcjeklc +++++ rm /tmp/tmp.C2smP0DWWc /tmp/tmp.J9Ajcjeklc +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XoOMumpX09 +++ mktemp ++ local LAST_ERR=/tmp/tmp.QImhGMcwgC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XoOMumpX09 ++ cat /tmp/tmp.QImhGMcwgC ++ rm /tmp/tmp.XoOMumpX09 /tmp/tmp.QImhGMcwgC ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v2Mn135qWN +++ mktemp ++ local LAST_ERR=/tmp/tmp.6hSUxDWxuf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.v2Mn135qWN ++ cat /tmp/tmp.6hSUxDWxuf ++ rm /tmp/tmp.v2Mn135qWN /tmp/tmp.6hSUxDWxuf ++ return 0 + state=ready ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NS5EpXYXiQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.OzQnrsxHB8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NS5EpXYXiQ ++ cat /tmp/tmp.OzQnrsxHB8 ++ rm /tmp/tmp.NS5EpXYXiQ /tmp/tmp.OzQnrsxHB8 ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.4ID4FADXnU ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.yFD6dV2rwK +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.4ID4FADXnU +++++ cat /tmp/tmp.yFD6dV2rwK +++++ rm /tmp/tmp.4ID4FADXnU /tmp/tmp.yFD6dV2rwK +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.k61jl5mjZS ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.txNfrImWc4 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.k61jl5mjZS +++++ cat /tmp/tmp.txNfrImWc4 +++++ rm /tmp/tmp.k61jl5mjZS /tmp/tmp.txNfrImWc4 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4QZLRvMJcd +++ mktemp ++ local LAST_ERR=/tmp/tmp.U7VtBtV59r ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4QZLRvMJcd ++ cat /tmp/tmp.U7VtBtV59r ++ rm /tmp/tmp.4QZLRvMJcd /tmp/tmp.U7VtBtV59r ++ return 0 + proxy_ready=3 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 3 == \3 ]] + echo + return + desc 'get replica cluster services endpoints' + set +o xtrace ----------------------------------------------------------------------------------- get replica cluster services endpoints ----------------------------------------------------------------------------------- ++ get_service_ip cross-site-replica-pxc-0 ++ local service=cross-site-replica-pxc-0 ++ grep -q NotFound ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.c6qGsPg3tq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6PuVlbU397 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.c6qGsPg3tq +++ cat /tmp/tmp.6PuVlbU397 +++ rm /tmp/tmp.c6qGsPg3tq /tmp/tmp.6PuVlbU397 +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xTUoXqYN94 +++ mktemp ++ local LAST_ERR=/tmp/tmp.VexIs6S2yb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xTUoXqYN94 ++ cat /tmp/tmp.VexIs6S2yb ++ rm /tmp/tmp.xTUoXqYN94 /tmp/tmp.VexIs6S2yb ++ return 0 ++ return + replica_endpoint0=34.118.239.36 ++ get_service_ip cross-site-replica-pxc-1 ++ local service=cross-site-replica-pxc-1 ++ grep -q NotFound ++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.McJiUvfhkC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XaZdQPCkes +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.McJiUvfhkC +++ cat /tmp/tmp.XaZdQPCkes +++ rm /tmp/tmp.McJiUvfhkC /tmp/tmp.XaZdQPCkes +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tRXUCPklfA +++ mktemp ++ local LAST_ERR=/tmp/tmp.JV3ckvwX7c ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tRXUCPklfA ++ cat /tmp/tmp.JV3ckvwX7c ++ rm /tmp/tmp.tRXUCPklfA /tmp/tmp.JV3ckvwX7c ++ return 0 ++ return + replica_endpoint1=34.118.228.40 ++ get_service_ip cross-site-replica-pxc-2 ++ local service=cross-site-replica-pxc-2 ++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sbAjJY20ko ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gGQjUDn8Rf +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.sbAjJY20ko +++ cat /tmp/tmp.gGQjUDn8Rf +++ rm /tmp/tmp.sbAjJY20ko /tmp/tmp.gGQjUDn8Rf +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Lxa3O90mKM +++ mktemp ++ local LAST_ERR=/tmp/tmp.31wZ6AoyKe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Lxa3O90mKM ++ cat /tmp/tmp.31wZ6AoyKe ++ rm /tmp/tmp.Lxa3O90mKM /tmp/tmp.31wZ6AoyKe ++ return 0 ++ return + replica_endpoint2=34.118.232.120 ++ run_mysql 'SELECT @@hostname hostname;' '-h cross-site-replica-proxysql -p3306 -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h cross-site-replica-proxysql -p3306 -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NJOguZJycs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TS4PMUev4y +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.NJOguZJycs +++ cat /tmp/tmp.TS4PMUev4y +++ rm /tmp/tmp.NJOguZJycs /tmp/tmp.TS4PMUev4y +++ return 0 ++ client_pod=pxc-client-56fd5498cd-f2ffc ++ wait_pod pxc-client-56fd5498cd-f2ffc ++ local pod=pxc-client-56fd5498cd-f2ffc ++ local max_retry=480 ++ local ns= +++ echo pxc-client-56fd5498cd-f2ffc +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ grep -E '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:07:42.190675 3234 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782961659699853000&timeoutSeconds=438&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok ++ set +o xtrace + replica_primary=cross-site-replica-pxc-0 ++ get_service_ip cross-site-replica-pxc-0 ++ local service=cross-site-replica-pxc-0 ++ grep -q NotFound ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mD4DvDWvxz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.eURy0MmjbX +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.mD4DvDWvxz +++ cat /tmp/tmp.eURy0MmjbX +++ rm /tmp/tmp.mD4DvDWvxz /tmp/tmp.eURy0MmjbX +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2M6S5jcFs0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.NDKcvpskVU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2M6S5jcFs0 ++ cat /tmp/tmp.NDKcvpskVU ++ rm /tmp/tmp.2M6S5jcFs0 /tmp/tmp.NDKcvpskVU ++ return 0 ++ return + replica_primary_endpoint=34.118.239.36 + run_mysql 'DELETE FROM myApp.myApp WHERE id=100500' '-h 34.118.239.36 -uroot -proot_password' + local 'command=DELETE FROM myApp.myApp WHERE id=100500' + local 'uri=-h 34.118.239.36 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.55sXw5Kq8k +++ mktemp ++ local LAST_ERR=/tmp/tmp.12GdrLSiIV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.55sXw5Kq8k ++ cat /tmp/tmp.12GdrLSiIV ++ rm /tmp/tmp.55sXw5Kq8k /tmp/tmp.12GdrLSiIV ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-f2ffc ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:07:59.849036 5993 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782961677574485000&timeoutSeconds=460&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + desc 'patch replica cluster with replicationChannels settings' + set +o xtrace ----------------------------------------------------------------------------------- patch replica cluster with replicationChannels settings ----------------------------------------------------------------------------------- + kubectl_bin patch pxc cross-site-replica --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "34.118.239.56", "port": 3306, "weight": 100},{"host": "34.118.235.139", "port": 3306, "weight": 100},{"host": "34.118.228.242", "port": 3306, "weight": 100}]}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.Pe51YQ87gE ++ mktemp + local LAST_ERR=/tmp/tmp.Inwt5YJqfy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-replica --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "34.118.239.56", "port": 3306, "weight": 100},{"host": "34.118.235.139", "port": 3306, "weight": 100},{"host": "34.118.228.242", "port": 3306, "weight": 100}]}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Pe51YQ87gE perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + cat /tmp/tmp.Inwt5YJqfy + rm /tmp/tmp.Pe51YQ87gE /tmp/tmp.Inwt5YJqfy + return 0 + desc 'patch replica cluster secrets with replication user' + set +o xtrace ----------------------------------------------------------------------------------- patch replica cluster secrets with replication user ----------------------------------------------------------------------------------- + patch_replication_secret cross-site-replica 3 3 + local cluster=cross-site-replica + local pxc_size=3 + local proxy_size=3 ++ echo -n new_password ++ base64 + kubectl_bin patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.0jReeQgMiN ++ mktemp + local LAST_ERR=/tmp/tmp.HhP4vW7Epd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0jReeQgMiN secret/my-cluster-secrets patched + cat /tmp/tmp.HhP4vW7Epd + rm /tmp/tmp.0jReeQgMiN /tmp/tmp.HhP4vW7Epd + return 0 + sleep 15 + wait_cluster_consistency cross-site-replica 3 3 + local cluster_name=cross-site-replica + local pxc_size=3 + local proxy_size=3 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/cross-site-replica to be ready' waiting for pxc/cross-site-replica to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.66gTtlKZaJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.jre9jVVYTc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.66gTtlKZaJ ++ cat /tmp/tmp.jre9jVVYTc ++ rm /tmp/tmp.66gTtlKZaJ /tmp/tmp.jre9jVVYTc ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EPUfGhx6lc +++ mktemp ++ local LAST_ERR=/tmp/tmp.R2VqWITGMF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EPUfGhx6lc ++ cat /tmp/tmp.R2VqWITGMF ++ rm /tmp/tmp.EPUfGhx6lc /tmp/tmp.R2VqWITGMF ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.gcvWJGUwJS ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.SzxazTtmpl +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.gcvWJGUwJS +++++ cat /tmp/tmp.SzxazTtmpl +++++ rm /tmp/tmp.gcvWJGUwJS /tmp/tmp.SzxazTtmpl +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.rXVjHS63H7 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ADdthQV3C5 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.rXVjHS63H7 +++++ cat /tmp/tmp.ADdthQV3C5 +++++ rm /tmp/tmp.rXVjHS63H7 /tmp/tmp.ADdthQV3C5 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DG1ZynWHMg +++ mktemp ++ local LAST_ERR=/tmp/tmp.JiwiT6UBdB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DG1ZynWHMg ++ cat /tmp/tmp.JiwiT6UBdB ++ rm /tmp/tmp.DG1ZynWHMg /tmp/tmp.JiwiT6UBdB ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xLkbV5OxFB +++ mktemp ++ local LAST_ERR=/tmp/tmp.78Z41fJhrV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xLkbV5OxFB ++ cat /tmp/tmp.78Z41fJhrV ++ rm /tmp/tmp.xLkbV5OxFB /tmp/tmp.78Z41fJhrV ++ return 0 + state=ready ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yUqGzglkYH +++ mktemp ++ local LAST_ERR=/tmp/tmp.GGOAQZLtzq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yUqGzglkYH ++ cat /tmp/tmp.GGOAQZLtzq ++ rm /tmp/tmp.yUqGzglkYH /tmp/tmp.GGOAQZLtzq ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.2rVpQqRVm3 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.biYl8l8Oxz +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.2rVpQqRVm3 +++++ cat /tmp/tmp.biYl8l8Oxz +++++ rm /tmp/tmp.2rVpQqRVm3 /tmp/tmp.biYl8l8Oxz +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.XEpcoZZsGC ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.SHQqyeEgKT +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.XEpcoZZsGC +++++ cat /tmp/tmp.SHQqyeEgKT +++++ rm /tmp/tmp.XEpcoZZsGC /tmp/tmp.SHQqyeEgKT +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZoX7Woqq0z +++ mktemp ++ local LAST_ERR=/tmp/tmp.fPtVzvihtb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZoX7Woqq0z ++ cat /tmp/tmp.fPtVzvihtb ++ rm /tmp/tmp.ZoX7Woqq0z /tmp/tmp.fPtVzvihtb ++ return 0 + proxy_ready=3 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 3 == \3 ]] + echo + return + wait_replication_channel 34.118.239.36 source_to_replica + local host=34.118.239.36 + local channel=source_to_replica + local retry=0 ++ run_mysql 'SELECT SERVICE_STATE FROM performance_schema.replication_connection_status WHERE CHANNEL_NAME='\''source_to_replica'\'';' '-h 34.118.239.36 -uroot -proot_password' + [[ ON == \O\N ]] + desc 'Check replication works between source -> replica' + set +o xtrace ----------------------------------------------------------------------------------- Check replication works between source -> replica ----------------------------------------------------------------------------------- + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 34.118.239.36 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 34.118.239.36 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-2-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 34.118.239.36 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 34.118.239.36 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fKQO1tfz1A +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ko74y6vh05 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fKQO1tfz1A ++ cat /tmp/tmp.Ko74y6vh05 ++ rm /tmp/tmp.fKQO1tfz1A /tmp/tmp.Ko74y6vh05 ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-f2ffc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:09:12.149601 16937 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782961749534696000&timeoutSeconds=328&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-2.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-2.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-2.sql /tmp/tmp.xyHcWb7plX/select-2.sql + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 34.118.228.40 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 34.118.228.40 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-2-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 34.118.228.40 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 34.118.228.40 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sbV2Jtz44r +++ mktemp ++ local LAST_ERR=/tmp/tmp.LekAcxsK9k ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sbV2Jtz44r ++ cat /tmp/tmp.LekAcxsK9k ++ rm /tmp/tmp.sbV2Jtz44r /tmp/tmp.LekAcxsK9k ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-f2ffc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:09:25.369241 18925 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782961762622635000&timeoutSeconds=547&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-2.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-2.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-2.sql /tmp/tmp.xyHcWb7plX/select-2.sql + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 34.118.232.120 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 34.118.232.120 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-2-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 34.118.232.120 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 34.118.232.120 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.i25VQJgABS +++ mktemp ++ local LAST_ERR=/tmp/tmp.wX9LN386DB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.i25VQJgABS ++ cat /tmp/tmp.wX9LN386DB ++ rm /tmp/tmp.i25VQJgABS /tmp/tmp.wX9LN386DB ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-56fd5498cd-f2ffc ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:09:41.495964 21148 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782961777943000000&timeoutSeconds=377&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-2.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-2.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-2.sql /tmp/tmp.xyHcWb7plX/select-2.sql + desc 'check that ProxySQL on replica cluster does not have writer hostgroup (11)' + set +o xtrace ----------------------------------------------------------------------------------- check that ProxySQL on replica cluster does not have writer hostgroup (11) ----------------------------------------------------------------------------------- + check_proxysql_hostgroup cross-site-replica 3 11 empty + local cluster=cross-site-replica + local size=3 + local hostgroup_id=11 + local expected=empty + local 'admin_uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' + local last_pod=2 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + local count ++ run_mysql_local 'SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-replica-proxysql-0 proxysql ++ local 'command=SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=cross-site-replica-proxysql-0 ++ local container_name=proxysql ++ set +o xtrace + count=0 + [[ empty == \e\m\p\t\y ]] + [[ 0 -ne 0 ]] + [[ empty == \p\o\p\u\l\a\t\e\d ]] + for i in '$(seq 0 $last_pod)' + local count ++ run_mysql_local 'SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-replica-proxysql-1 proxysql ++ local 'command=SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=cross-site-replica-proxysql-1 ++ local container_name=proxysql ++ set +o xtrace + count=0 + [[ empty == \e\m\p\t\y ]] + [[ 0 -ne 0 ]] + [[ empty == \p\o\p\u\l\a\t\e\d ]] + for i in '$(seq 0 $last_pod)' + local count ++ run_mysql_local 'SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-replica-proxysql-2 proxysql ++ local 'command=SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=cross-site-replica-proxysql-2 ++ local container_name=proxysql ++ set +o xtrace + count=0 + [[ empty == \e\m\p\t\y ]] + [[ 0 -ne 0 ]] + [[ empty == \p\o\p\u\l\a\t\e\d ]] + desc 'verify cluster stability and no ProxySQL pod restarts over 60 seconds' + set +o xtrace ----------------------------------------------------------------------------------- verify cluster stability and no ProxySQL pod restarts over 60 seconds ----------------------------------------------------------------------------------- + proxysql_restarts_before=() + local proxysql_restarts_before + for i in 0 1 2 + proxysql_restarts_before+=("$(kubectl_bin get pod "${replica_cluster}-proxysql-${i}" -o jsonpath='{.status.containerStatuses[?(@.name=="proxysql")].restartCount}')") ++ kubectl_bin get pod cross-site-replica-proxysql-0 -o 'jsonpath={.status.containerStatuses[?(@.name=="proxysql")].restartCount}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OfYFi3Kagd +++ mktemp ++ local LAST_ERR=/tmp/tmp.OE67S2wrv5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod cross-site-replica-proxysql-0 -o 'jsonpath={.status.containerStatuses[?(@.name=="proxysql")].restartCount}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OfYFi3Kagd ++ cat /tmp/tmp.OE67S2wrv5 ++ rm /tmp/tmp.OfYFi3Kagd /tmp/tmp.OE67S2wrv5 ++ return 0 + for i in 0 1 2 + proxysql_restarts_before+=("$(kubectl_bin get pod "${replica_cluster}-proxysql-${i}" -o jsonpath='{.status.containerStatuses[?(@.name=="proxysql")].restartCount}')") ++ kubectl_bin get pod cross-site-replica-proxysql-1 -o 'jsonpath={.status.containerStatuses[?(@.name=="proxysql")].restartCount}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.apbmzCcWnG +++ mktemp ++ local LAST_ERR=/tmp/tmp.cB38NwgvA7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod cross-site-replica-proxysql-1 -o 'jsonpath={.status.containerStatuses[?(@.name=="proxysql")].restartCount}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.apbmzCcWnG ++ cat /tmp/tmp.cB38NwgvA7 ++ rm /tmp/tmp.apbmzCcWnG /tmp/tmp.cB38NwgvA7 ++ return 0 + for i in 0 1 2 + proxysql_restarts_before+=("$(kubectl_bin get pod "${replica_cluster}-proxysql-${i}" -o jsonpath='{.status.containerStatuses[?(@.name=="proxysql")].restartCount}')") ++ kubectl_bin get pod cross-site-replica-proxysql-2 -o 'jsonpath={.status.containerStatuses[?(@.name=="proxysql")].restartCount}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BPDCVPzjAB +++ mktemp ++ local LAST_ERR=/tmp/tmp.6TSRiy6IRP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod cross-site-replica-proxysql-2 -o 'jsonpath={.status.containerStatuses[?(@.name=="proxysql")].restartCount}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BPDCVPzjAB ++ cat /tmp/tmp.6TSRiy6IRP ++ rm /tmp/tmp.BPDCVPzjAB /tmp/tmp.6TSRiy6IRP ++ return 0 + sleep 60 + wait_cluster_consistency cross-site-replica 3 3 + local cluster_name=cross-site-replica + local pxc_size=3 + local proxy_size=3 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/cross-site-replica to be ready' waiting for pxc/cross-site-replica to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u6vpm2jhuC +++ mktemp ++ local LAST_ERR=/tmp/tmp.j1jPQVgEuL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u6vpm2jhuC ++ cat /tmp/tmp.j1jPQVgEuL ++ rm /tmp/tmp.u6vpm2jhuC /tmp/tmp.j1jPQVgEuL ++ return 0 + state=ready ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8kRJWpmUwS +++ mktemp ++ local LAST_ERR=/tmp/tmp.JLtap8qnFq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8kRJWpmUwS ++ cat /tmp/tmp.JLtap8qnFq ++ rm /tmp/tmp.8kRJWpmUwS /tmp/tmp.JLtap8qnFq ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.nSKjFGlP1l ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.I2CSjCD2sI +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.nSKjFGlP1l +++++ cat /tmp/tmp.I2CSjCD2sI +++++ rm /tmp/tmp.nSKjFGlP1l /tmp/tmp.I2CSjCD2sI +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.7Nr8lPOLTz ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.WlNjQg8ufE +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.7Nr8lPOLTz +++++ cat /tmp/tmp.WlNjQg8ufE +++++ rm /tmp/tmp.7Nr8lPOLTz /tmp/tmp.WlNjQg8ufE +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IR5ingiQey +++ mktemp ++ local LAST_ERR=/tmp/tmp.vSe4RfQw8R ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IR5ingiQey ++ cat /tmp/tmp.vSe4RfQw8R ++ rm /tmp/tmp.IR5ingiQey /tmp/tmp.vSe4RfQw8R ++ return 0 + proxy_ready=3 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 3 == \3 ]] + echo + return + for i in 0 1 2 ++ kubectl_bin get pod cross-site-replica-proxysql-0 -o 'jsonpath={.status.containerStatuses[?(@.name=="proxysql")].restartCount}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dPh75v71rS +++ mktemp ++ local LAST_ERR=/tmp/tmp.E3MsSM7cV4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod cross-site-replica-proxysql-0 -o 'jsonpath={.status.containerStatuses[?(@.name=="proxysql")].restartCount}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dPh75v71rS ++ cat /tmp/tmp.E3MsSM7cV4 ++ rm /tmp/tmp.dPh75v71rS /tmp/tmp.E3MsSM7cV4 ++ return 0 + restarts_after=0 + [[ 0 -gt 0 ]] + for i in 0 1 2 ++ kubectl_bin get pod cross-site-replica-proxysql-1 -o 'jsonpath={.status.containerStatuses[?(@.name=="proxysql")].restartCount}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ILYmoOqtKe +++ mktemp ++ local LAST_ERR=/tmp/tmp.XkvSq2pAeF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod cross-site-replica-proxysql-1 -o 'jsonpath={.status.containerStatuses[?(@.name=="proxysql")].restartCount}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ILYmoOqtKe ++ cat /tmp/tmp.XkvSq2pAeF ++ rm /tmp/tmp.ILYmoOqtKe /tmp/tmp.XkvSq2pAeF ++ return 0 + restarts_after=0 + [[ 0 -gt 0 ]] + for i in 0 1 2 ++ kubectl_bin get pod cross-site-replica-proxysql-2 -o 'jsonpath={.status.containerStatuses[?(@.name=="proxysql")].restartCount}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5OkHmfz7eI +++ mktemp ++ local LAST_ERR=/tmp/tmp.xhA1srQrQz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pod cross-site-replica-proxysql-2 -o 'jsonpath={.status.containerStatuses[?(@.name=="proxysql")].restartCount}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5OkHmfz7eI ++ cat /tmp/tmp.xhA1srQrQz ++ rm /tmp/tmp.5OkHmfz7eI /tmp/tmp.xhA1srQrQz ++ return 0 + restarts_after=0 + [[ 0 -gt 0 ]] + desc 'check read access via ProxySQL on replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- check read access via ProxySQL on replica cluster ----------------------------------------------------------------------------------- + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h cross-site-replica-proxysql -p3306 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h cross-site-replica-proxysql -p3306 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-2-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h cross-site-replica-proxysql -p3306 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h cross-site-replica-proxysql -p3306 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.b2d3qUIQXs +++ mktemp ++ local LAST_ERR=/tmp/tmp.bh33bpfSoa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.b2d3qUIQXs ++ cat /tmp/tmp.bh33bpfSoa ++ rm /tmp/tmp.b2d3qUIQXs /tmp/tmp.bh33bpfSoa ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-56fd5498cd-f2ffc ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:11:41.547597 6344 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782961898435705000&timeoutSeconds=451&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-2.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-2.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-2.sql /tmp/tmp.xyHcWb7plX/select-2.sql + run_backup cross-site-replica backup-minio-replica + local cluster=cross-site-replica + local backup=backup-minio-replica + local status=Succeeded + log 'run pxc-backup/backup-minio-replica' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T03:11:50+0000]' run pxc-backup/backup-minio-replica [2026-07-02T03:11:50+0000] run pxc-backup/backup-minio-replica + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/backup-minio-replica.yml ++ mktemp + local LAST_OUT=/tmp/tmp.nXPzUgxMZ4 ++ mktemp + local LAST_ERR=/tmp/tmp.KfAM5VvHvQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/backup-minio-replica.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nXPzUgxMZ4 perconaxtradbclusterbackup.pxc.percona.com/backup-minio-replica created + cat /tmp/tmp.KfAM5VvHvQ + rm /tmp/tmp.nXPzUgxMZ4 /tmp/tmp.KfAM5VvHvQ + return 0 + wait_backup backup-minio-replica Succeeded + local backup=backup-minio-replica + local status=Succeeded + set +o xtrace waiting for pxc-backup/backup-minio-replica to reach Succeeded state...........Succeeded + desc 'Switch clusters over' + set +o xtrace ----------------------------------------------------------------------------------- Switch clusters over ----------------------------------------------------------------------------------- + switch_namespace cross-site-proxysql-28035 + local ns=cross-site-proxysql-28035 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.SyG2xGz2mD +++ mktemp ++ local LAST_ERR=/tmp/tmp.okbJt1YgDW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SyG2xGz2mD ++ cat /tmp/tmp.okbJt1YgDW ++ rm /tmp/tmp.SyG2xGz2mD /tmp/tmp.okbJt1YgDW ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-28035 ++ mktemp + local LAST_OUT=/tmp/tmp.bH2fsyIm7T ++ mktemp + local LAST_ERR=/tmp/tmp.HKRcu2KHUS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-28035 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bH2fsyIm7T Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7" modified. + cat /tmp/tmp.HKRcu2KHUS + rm /tmp/tmp.bH2fsyIm7T /tmp/tmp.HKRcu2KHUS + return 0 + desc 'rebuild source cluster' + set +o xtrace ----------------------------------------------------------------------------------- rebuild source cluster ----------------------------------------------------------------------------------- ++ kubectl_bin get -n cross-site-proxysql-replica-17697 pxc-backup backup-minio-replica -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TaNYPMP0Yi +++ mktemp ++ local LAST_ERR=/tmp/tmp.7k0gpaYuyT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get -n cross-site-proxysql-replica-17697 pxc-backup backup-minio-replica -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TaNYPMP0Yi ++ cat /tmp/tmp.7k0gpaYuyT ++ rm /tmp/tmp.TaNYPMP0Yi /tmp/tmp.7k0gpaYuyT ++ return 0 + destination=s3://operator-testing/cross-site-replica-2026-07-02-03:11:53-full + /usr/bin/sed -e s~#cluster~cross-site-source~ + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/conf/restore-backup-minio.yml + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-proxysql-28035~ + /usr/bin/sed -e s~#destination~s3://operator-testing/cross-site-replica-2026-07-02-03:11:53-full~ + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.fxZX5oXLHv ++ mktemp + local LAST_ERR=/tmp/tmp.oHKQ4nnmzs + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fxZX5oXLHv perconaxtradbclusterrestore.pxc.percona.com/backup-minio created + cat /tmp/tmp.oHKQ4nnmzs + rm /tmp/tmp.fxZX5oXLHv /tmp/tmp.oHKQ4nnmzs + return 0 + wait_cluster_consistency cross-site-source 3 3 + local cluster_name=cross-site-source + local pxc_size=3 + local proxy_size=3 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/cross-site-source to be ready' waiting for pxc/cross-site-source to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rNWmqeLeTn +++ mktemp ++ local LAST_ERR=/tmp/tmp.9mMTjvSOZu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rNWmqeLeTn ++ cat /tmp/tmp.9mMTjvSOZu ++ rm /tmp/tmp.rNWmqeLeTn /tmp/tmp.9mMTjvSOZu ++ return 0 + state=stopping ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yz8N1hGdKG +++ mktemp ++ local LAST_ERR=/tmp/tmp.vkj1lOBSjV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yz8N1hGdKG ++ cat /tmp/tmp.vkj1lOBSjV ++ rm /tmp/tmp.yz8N1hGdKG /tmp/tmp.vkj1lOBSjV ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.NOSCOfgWqQ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.4Aoiwwu7Ht +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.NOSCOfgWqQ +++++ cat /tmp/tmp.4Aoiwwu7Ht +++++ rm /tmp/tmp.NOSCOfgWqQ /tmp/tmp.4Aoiwwu7Ht +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.KmicnQslAo ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.U6tFyuCFuW +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.KmicnQslAo +++++ cat /tmp/tmp.U6tFyuCFuW +++++ rm /tmp/tmp.KmicnQslAo /tmp/tmp.U6tFyuCFuW +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6AN2htCKkD +++ mktemp ++ local LAST_ERR=/tmp/tmp.G5QSsUFrnd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6AN2htCKkD ++ cat /tmp/tmp.G5QSsUFrnd ++ rm /tmp/tmp.6AN2htCKkD /tmp/tmp.G5QSsUFrnd ++ return 0 + proxy_ready= + [[ stopping == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DNbG1J0xBF +++ mktemp ++ local LAST_ERR=/tmp/tmp.gehWzzOzLc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DNbG1J0xBF ++ cat /tmp/tmp.gehWzzOzLc ++ rm /tmp/tmp.DNbG1J0xBF /tmp/tmp.gehWzzOzLc ++ return 0 + state=stopping ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.m14cV1LPnx +++ mktemp ++ local LAST_ERR=/tmp/tmp.Nqrq8ouRSD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.m14cV1LPnx ++ cat /tmp/tmp.Nqrq8ouRSD ++ rm /tmp/tmp.m14cV1LPnx /tmp/tmp.Nqrq8ouRSD ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ahHGXDpeK1 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Zdh1ad6tOC +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ahHGXDpeK1 +++++ cat /tmp/tmp.Zdh1ad6tOC +++++ rm /tmp/tmp.ahHGXDpeK1 /tmp/tmp.Zdh1ad6tOC +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.2C8l0FVXgU ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.6LdS2dqhKF +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.2C8l0FVXgU +++++ cat /tmp/tmp.6LdS2dqhKF +++++ rm /tmp/tmp.2C8l0FVXgU /tmp/tmp.6LdS2dqhKF +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OghPqAzWRG +++ mktemp ++ local LAST_ERR=/tmp/tmp.wUaCcT6sJw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OghPqAzWRG ++ cat /tmp/tmp.wUaCcT6sJw ++ rm /tmp/tmp.OghPqAzWRG /tmp/tmp.wUaCcT6sJw ++ return 0 + proxy_ready= + [[ stopping == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JuBQxODxHo +++ mktemp ++ local LAST_ERR=/tmp/tmp.vr8oTRLvmt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JuBQxODxHo ++ cat /tmp/tmp.vr8oTRLvmt ++ rm /tmp/tmp.JuBQxODxHo /tmp/tmp.vr8oTRLvmt ++ return 0 + state=stopping ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.s0RGQgAxG3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mzuzjQUuep ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.s0RGQgAxG3 ++ cat /tmp/tmp.mzuzjQUuep ++ rm /tmp/tmp.s0RGQgAxG3 /tmp/tmp.mzuzjQUuep ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.P26DCJHcv1 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.sHpGn4VTEq +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.P26DCJHcv1 +++++ cat /tmp/tmp.sHpGn4VTEq +++++ rm /tmp/tmp.P26DCJHcv1 /tmp/tmp.sHpGn4VTEq +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.jWr86hAPwz ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.YZEmgPA3L2 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.jWr86hAPwz +++++ cat /tmp/tmp.YZEmgPA3L2 +++++ rm /tmp/tmp.jWr86hAPwz /tmp/tmp.YZEmgPA3L2 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6ddX3y0w2g +++ mktemp ++ local LAST_ERR=/tmp/tmp.G7aV1WOlJg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6ddX3y0w2g ++ cat /tmp/tmp.G7aV1WOlJg ++ rm /tmp/tmp.6ddX3y0w2g /tmp/tmp.G7aV1WOlJg ++ return 0 + proxy_ready= + [[ stopping == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YYY33yygAD +++ mktemp ++ local LAST_ERR=/tmp/tmp.GXDAW6SdUn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YYY33yygAD ++ cat /tmp/tmp.GXDAW6SdUn ++ rm /tmp/tmp.YYY33yygAD /tmp/tmp.GXDAW6SdUn ++ return 0 + state=stopping ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6v5Rn9FaSz +++ mktemp ++ local LAST_ERR=/tmp/tmp.BOTBbXKoAx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6v5Rn9FaSz ++ cat /tmp/tmp.BOTBbXKoAx ++ rm /tmp/tmp.6v5Rn9FaSz /tmp/tmp.BOTBbXKoAx ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.IgyJ4Qgw7F ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.vRKLhsfwY6 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.IgyJ4Qgw7F +++++ cat /tmp/tmp.vRKLhsfwY6 +++++ rm /tmp/tmp.IgyJ4Qgw7F /tmp/tmp.vRKLhsfwY6 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.8rByBos1f4 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.w3fgtU3DxQ +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.8rByBos1f4 +++++ cat /tmp/tmp.w3fgtU3DxQ +++++ rm /tmp/tmp.8rByBos1f4 /tmp/tmp.w3fgtU3DxQ +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Wgk4wfi8eX +++ mktemp ++ local LAST_ERR=/tmp/tmp.TBpW540ESQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Wgk4wfi8eX ++ cat /tmp/tmp.TBpW540ESQ ++ rm /tmp/tmp.Wgk4wfi8eX /tmp/tmp.TBpW540ESQ ++ return 0 + proxy_ready= + [[ stopping == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TjxmIS45Ue +++ mktemp ++ local LAST_ERR=/tmp/tmp.8d6zo9CymN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TjxmIS45Ue ++ cat /tmp/tmp.8d6zo9CymN ++ rm /tmp/tmp.TjxmIS45Ue /tmp/tmp.8d6zo9CymN ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3uCdjHtDob +++ mktemp ++ local LAST_ERR=/tmp/tmp.AhF3WPXAyz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3uCdjHtDob ++ cat /tmp/tmp.AhF3WPXAyz ++ rm /tmp/tmp.3uCdjHtDob /tmp/tmp.AhF3WPXAyz ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.FLqSpXMDVS ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ysoGu9pQov +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.FLqSpXMDVS +++++ cat /tmp/tmp.ysoGu9pQov +++++ rm /tmp/tmp.FLqSpXMDVS /tmp/tmp.ysoGu9pQov +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.9p9fiUxH8N ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.JpKIGrfxLU +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.9p9fiUxH8N +++++ cat /tmp/tmp.JpKIGrfxLU +++++ rm /tmp/tmp.9p9fiUxH8N /tmp/tmp.JpKIGrfxLU +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5TDbYvImhm +++ mktemp ++ local LAST_ERR=/tmp/tmp.O46Z8VWQpw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5TDbYvImhm ++ cat /tmp/tmp.O46Z8VWQpw ++ rm /tmp/tmp.5TDbYvImhm /tmp/tmp.O46Z8VWQpw ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VDfkHlsbhz +++ mktemp ++ local LAST_ERR=/tmp/tmp.MGuGiL3heL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VDfkHlsbhz ++ cat /tmp/tmp.MGuGiL3heL ++ rm /tmp/tmp.VDfkHlsbhz /tmp/tmp.MGuGiL3heL ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VVSidYeWbG +++ mktemp ++ local LAST_ERR=/tmp/tmp.IwO4JJN7kc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VVSidYeWbG ++ cat /tmp/tmp.IwO4JJN7kc ++ rm /tmp/tmp.VVSidYeWbG /tmp/tmp.IwO4JJN7kc ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.YYvMkfBhIe ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.0XGuAUQb5t +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.YYvMkfBhIe +++++ cat /tmp/tmp.0XGuAUQb5t +++++ rm /tmp/tmp.YYvMkfBhIe /tmp/tmp.0XGuAUQb5t +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.r62tY4ttrv ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.uLicfY6mw7 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.r62tY4ttrv +++++ cat /tmp/tmp.uLicfY6mw7 +++++ rm /tmp/tmp.r62tY4ttrv /tmp/tmp.uLicfY6mw7 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZW0mKfZQeh +++ mktemp ++ local LAST_ERR=/tmp/tmp.BKyXPJi1Vi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZW0mKfZQeh ++ cat /tmp/tmp.BKyXPJi1Vi ++ rm /tmp/tmp.ZW0mKfZQeh /tmp/tmp.BKyXPJi1Vi ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GRi27K6EhQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.9y4iDU39ci ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GRi27K6EhQ ++ cat /tmp/tmp.9y4iDU39ci ++ rm /tmp/tmp.GRi27K6EhQ /tmp/tmp.9y4iDU39ci ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7KNKW8ELdG +++ mktemp ++ local LAST_ERR=/tmp/tmp.2fITPufK4W ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7KNKW8ELdG ++ cat /tmp/tmp.2fITPufK4W ++ rm /tmp/tmp.7KNKW8ELdG /tmp/tmp.2fITPufK4W ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.tLdX4hZBah ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.sV7DIE0Imw +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.tLdX4hZBah +++++ cat /tmp/tmp.sV7DIE0Imw +++++ rm /tmp/tmp.tLdX4hZBah /tmp/tmp.sV7DIE0Imw +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.sHolDPJqQ2 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.xR51EDPpL7 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.sHolDPJqQ2 +++++ cat /tmp/tmp.xR51EDPpL7 +++++ rm /tmp/tmp.sHolDPJqQ2 /tmp/tmp.xR51EDPpL7 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sOtogFKAm7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.pgIZ1bVE04 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sOtogFKAm7 ++ cat /tmp/tmp.pgIZ1bVE04 ++ rm /tmp/tmp.sOtogFKAm7 /tmp/tmp.pgIZ1bVE04 ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eGCngBMjfm +++ mktemp ++ local LAST_ERR=/tmp/tmp.GYyr9DN2vN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eGCngBMjfm ++ cat /tmp/tmp.GYyr9DN2vN ++ rm /tmp/tmp.eGCngBMjfm /tmp/tmp.GYyr9DN2vN ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HXzPNvCJ29 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7MEumrXiMa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HXzPNvCJ29 ++ cat /tmp/tmp.7MEumrXiMa ++ rm /tmp/tmp.HXzPNvCJ29 /tmp/tmp.7MEumrXiMa ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.awzNLuwiSg ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.jyLneHpOZw +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.awzNLuwiSg +++++ cat /tmp/tmp.jyLneHpOZw +++++ rm /tmp/tmp.awzNLuwiSg /tmp/tmp.jyLneHpOZw +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.6iNEibRcUx ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.qYrhcVB3he +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.6iNEibRcUx +++++ cat /tmp/tmp.qYrhcVB3he +++++ rm /tmp/tmp.6iNEibRcUx /tmp/tmp.qYrhcVB3he +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Qdb7WPmnAO +++ mktemp ++ local LAST_ERR=/tmp/tmp.fRsdEihNxw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Qdb7WPmnAO ++ cat /tmp/tmp.fRsdEihNxw ++ rm /tmp/tmp.Qdb7WPmnAO /tmp/tmp.fRsdEihNxw ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gmdLHAnH6s +++ mktemp ++ local LAST_ERR=/tmp/tmp.lWKwAEAieT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gmdLHAnH6s ++ cat /tmp/tmp.lWKwAEAieT ++ rm /tmp/tmp.gmdLHAnH6s /tmp/tmp.lWKwAEAieT ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4nmrYU2mPy +++ mktemp ++ local LAST_ERR=/tmp/tmp.o4Ak7VkRYx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4nmrYU2mPy ++ cat /tmp/tmp.o4Ak7VkRYx ++ rm /tmp/tmp.4nmrYU2mPy /tmp/tmp.o4Ak7VkRYx ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.u53Dsum95Q ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.YTmZkl8fbM +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.u53Dsum95Q +++++ cat /tmp/tmp.YTmZkl8fbM +++++ rm /tmp/tmp.u53Dsum95Q /tmp/tmp.YTmZkl8fbM +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.k1QGfTJEmh ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.nGyaPAsTPA +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.k1QGfTJEmh +++++ cat /tmp/tmp.nGyaPAsTPA +++++ rm /tmp/tmp.k1QGfTJEmh /tmp/tmp.nGyaPAsTPA +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rwIbvF9oIj +++ mktemp ++ local LAST_ERR=/tmp/tmp.k52WfUJx17 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rwIbvF9oIj ++ cat /tmp/tmp.k52WfUJx17 ++ rm /tmp/tmp.rwIbvF9oIj /tmp/tmp.k52WfUJx17 ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Asx3ZcvNvu +++ mktemp ++ local LAST_ERR=/tmp/tmp.g6O0qg0v42 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Asx3ZcvNvu ++ cat /tmp/tmp.g6O0qg0v42 ++ rm /tmp/tmp.Asx3ZcvNvu /tmp/tmp.g6O0qg0v42 ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8tbLxDhimR +++ mktemp ++ local LAST_ERR=/tmp/tmp.pQyXTeC6Bw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8tbLxDhimR ++ cat /tmp/tmp.pQyXTeC6Bw ++ rm /tmp/tmp.8tbLxDhimR /tmp/tmp.pQyXTeC6Bw ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.8dhayGO6Qq ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.GHBL9xEzYs +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.8dhayGO6Qq +++++ cat /tmp/tmp.GHBL9xEzYs +++++ rm /tmp/tmp.8dhayGO6Qq /tmp/tmp.GHBL9xEzYs +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.GCvdusNBmZ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.gw8XCZFq0B +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.GCvdusNBmZ +++++ cat /tmp/tmp.gw8XCZFq0B +++++ rm /tmp/tmp.GCvdusNBmZ /tmp/tmp.gw8XCZFq0B +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WfwHnMBxoY +++ mktemp ++ local LAST_ERR=/tmp/tmp.VKC0xISaRh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WfwHnMBxoY ++ cat /tmp/tmp.VKC0xISaRh ++ rm /tmp/tmp.WfwHnMBxoY /tmp/tmp.VKC0xISaRh ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pfwkx2E5p5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.eqNVyQpbVP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pfwkx2E5p5 ++ cat /tmp/tmp.eqNVyQpbVP ++ rm /tmp/tmp.pfwkx2E5p5 /tmp/tmp.eqNVyQpbVP ++ return 0 + state=paused ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TPjVZckHwe +++ mktemp ++ local LAST_ERR=/tmp/tmp.Bqp7MuivMa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TPjVZckHwe ++ cat /tmp/tmp.Bqp7MuivMa ++ rm /tmp/tmp.TPjVZckHwe /tmp/tmp.Bqp7MuivMa ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.BlXV7Wu5KJ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.I434pJFrJK +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.BlXV7Wu5KJ +++++ cat /tmp/tmp.I434pJFrJK +++++ rm /tmp/tmp.BlXV7Wu5KJ /tmp/tmp.I434pJFrJK +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.x3FbqxZn7B ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.p02gkDOVQp +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.x3FbqxZn7B +++++ cat /tmp/tmp.p02gkDOVQp +++++ rm /tmp/tmp.x3FbqxZn7B /tmp/tmp.p02gkDOVQp +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2yV7ogww4L +++ mktemp ++ local LAST_ERR=/tmp/tmp.QoYY0iRyCb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2yV7ogww4L ++ cat /tmp/tmp.QoYY0iRyCb ++ rm /tmp/tmp.2yV7ogww4L /tmp/tmp.QoYY0iRyCb ++ return 0 + proxy_ready= + [[ paused == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qyudu9DJtz +++ mktemp ++ local LAST_ERR=/tmp/tmp.xTd60DYZsT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qyudu9DJtz ++ cat /tmp/tmp.xTd60DYZsT ++ rm /tmp/tmp.qyudu9DJtz /tmp/tmp.xTd60DYZsT ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tMgBTIwZSa +++ mktemp ++ local LAST_ERR=/tmp/tmp.FzX6ge9EM5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tMgBTIwZSa ++ cat /tmp/tmp.FzX6ge9EM5 ++ rm /tmp/tmp.tMgBTIwZSa /tmp/tmp.FzX6ge9EM5 ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.JxIScfljfO ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.OotUNHD2lX +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.JxIScfljfO +++++ cat /tmp/tmp.OotUNHD2lX +++++ rm /tmp/tmp.JxIScfljfO /tmp/tmp.OotUNHD2lX +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.iSbQ6ozhRe ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.IPNJhQprA1 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.iSbQ6ozhRe +++++ cat /tmp/tmp.IPNJhQprA1 +++++ rm /tmp/tmp.iSbQ6ozhRe /tmp/tmp.IPNJhQprA1 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wnZpdlCOPv +++ mktemp ++ local LAST_ERR=/tmp/tmp.nsS7PlhARC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wnZpdlCOPv ++ cat /tmp/tmp.nsS7PlhARC ++ rm /tmp/tmp.wnZpdlCOPv /tmp/tmp.nsS7PlhARC ++ return 0 + proxy_ready= + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1eB1DXOEAA +++ mktemp ++ local LAST_ERR=/tmp/tmp.qwV73EORhX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1eB1DXOEAA ++ cat /tmp/tmp.qwV73EORhX ++ rm /tmp/tmp.1eB1DXOEAA /tmp/tmp.qwV73EORhX ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZU9QQX5lii +++ mktemp ++ local LAST_ERR=/tmp/tmp.mmJBWLtCHN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZU9QQX5lii ++ cat /tmp/tmp.mmJBWLtCHN ++ rm /tmp/tmp.ZU9QQX5lii /tmp/tmp.mmJBWLtCHN ++ return 0 + pxc_ready= + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.eIiQIdslv2 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.CiecKynjW9 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.eIiQIdslv2 +++++ cat /tmp/tmp.CiecKynjW9 +++++ rm /tmp/tmp.eIiQIdslv2 /tmp/tmp.CiecKynjW9 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.7yMIYelm3o ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.p5Glg6pIdO +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.7yMIYelm3o +++++ cat /tmp/tmp.p5Glg6pIdO +++++ rm /tmp/tmp.7yMIYelm3o /tmp/tmp.p5Glg6pIdO +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oB8NZ0NtDW +++ mktemp ++ local LAST_ERR=/tmp/tmp.QRxfkJXq7Z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oB8NZ0NtDW ++ cat /tmp/tmp.QRxfkJXq7Z ++ rm /tmp/tmp.oB8NZ0NtDW /tmp/tmp.QRxfkJXq7Z ++ return 0 + proxy_ready=1 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cmYjygnFiI +++ mktemp ++ local LAST_ERR=/tmp/tmp.7fcKi57Q2i ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cmYjygnFiI ++ cat /tmp/tmp.7fcKi57Q2i ++ rm /tmp/tmp.cmYjygnFiI /tmp/tmp.7fcKi57Q2i ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ifoPfT6kVg +++ mktemp ++ local LAST_ERR=/tmp/tmp.09bVsopsq2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ifoPfT6kVg ++ cat /tmp/tmp.09bVsopsq2 ++ rm /tmp/tmp.ifoPfT6kVg /tmp/tmp.09bVsopsq2 ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.4rFHPeOjzP ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Xo7pmnkAoE +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.4rFHPeOjzP +++++ cat /tmp/tmp.Xo7pmnkAoE +++++ rm /tmp/tmp.4rFHPeOjzP /tmp/tmp.Xo7pmnkAoE +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.g58MJjYQAC ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.qG2h2Xmj6H +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.g58MJjYQAC +++++ cat /tmp/tmp.qG2h2Xmj6H +++++ rm /tmp/tmp.g58MJjYQAC /tmp/tmp.qG2h2Xmj6H +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fIEKjMnRAy +++ mktemp ++ local LAST_ERR=/tmp/tmp.GT5bGsg1YF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fIEKjMnRAy ++ cat /tmp/tmp.GT5bGsg1YF ++ rm /tmp/tmp.fIEKjMnRAy /tmp/tmp.GT5bGsg1YF ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZhOxiorYvE +++ mktemp ++ local LAST_ERR=/tmp/tmp.OJC9Oocoaq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZhOxiorYvE ++ cat /tmp/tmp.OJC9Oocoaq ++ rm /tmp/tmp.ZhOxiorYvE /tmp/tmp.OJC9Oocoaq ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nJ6xaY4oZn +++ mktemp ++ local LAST_ERR=/tmp/tmp.DAa4LG9XE8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nJ6xaY4oZn ++ cat /tmp/tmp.DAa4LG9XE8 ++ rm /tmp/tmp.nJ6xaY4oZn /tmp/tmp.DAa4LG9XE8 ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.D8E1o9cyHY ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.y7KjncA8Lc +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.D8E1o9cyHY +++++ cat /tmp/tmp.y7KjncA8Lc +++++ rm /tmp/tmp.D8E1o9cyHY /tmp/tmp.y7KjncA8Lc +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.aBuXRre4pS ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.CloZPOUtQF +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.aBuXRre4pS +++++ cat /tmp/tmp.CloZPOUtQF +++++ rm /tmp/tmp.aBuXRre4pS /tmp/tmp.CloZPOUtQF +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0Dnwi2JPpZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.BQerrOzT1U ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0Dnwi2JPpZ ++ cat /tmp/tmp.BQerrOzT1U ++ rm /tmp/tmp.0Dnwi2JPpZ /tmp/tmp.BQerrOzT1U ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.atjGQdeFF9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.3cZFr15xKH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.atjGQdeFF9 ++ cat /tmp/tmp.3cZFr15xKH ++ rm /tmp/tmp.atjGQdeFF9 /tmp/tmp.3cZFr15xKH ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gfVFBRJkp1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.AWW57SWDqo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gfVFBRJkp1 ++ cat /tmp/tmp.AWW57SWDqo ++ rm /tmp/tmp.gfVFBRJkp1 /tmp/tmp.AWW57SWDqo ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.HXuPxYmaw1 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.CQQDUowCOY +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.HXuPxYmaw1 +++++ cat /tmp/tmp.CQQDUowCOY +++++ rm /tmp/tmp.HXuPxYmaw1 /tmp/tmp.CQQDUowCOY +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.W4F1XfYkwU ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.4hhdzaUXwp +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.W4F1XfYkwU +++++ cat /tmp/tmp.4hhdzaUXwp +++++ rm /tmp/tmp.W4F1XfYkwU /tmp/tmp.4hhdzaUXwp +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n0X7uJHKp8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.uOdNL1gjFw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.n0X7uJHKp8 ++ cat /tmp/tmp.uOdNL1gjFw ++ rm /tmp/tmp.n0X7uJHKp8 /tmp/tmp.uOdNL1gjFw ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Yw95SFKWlT +++ mktemp ++ local LAST_ERR=/tmp/tmp.dY3vC7IvKX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Yw95SFKWlT ++ cat /tmp/tmp.dY3vC7IvKX ++ rm /tmp/tmp.Yw95SFKWlT /tmp/tmp.dY3vC7IvKX ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.50ZyA7lUTm +++ mktemp ++ local LAST_ERR=/tmp/tmp.SMxWUxGdpg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.50ZyA7lUTm ++ cat /tmp/tmp.SMxWUxGdpg ++ rm /tmp/tmp.50ZyA7lUTm /tmp/tmp.SMxWUxGdpg ++ return 0 + pxc_ready=1 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.4SI7RoIUd4 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.eqhCSLcRXj +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.4SI7RoIUd4 +++++ cat /tmp/tmp.eqhCSLcRXj +++++ rm /tmp/tmp.4SI7RoIUd4 /tmp/tmp.eqhCSLcRXj +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.gezq1ua38j ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.CfcJpMZv8k +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.gezq1ua38j +++++ cat /tmp/tmp.CfcJpMZv8k +++++ rm /tmp/tmp.gezq1ua38j /tmp/tmp.CfcJpMZv8k +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SDCFunQTOH +++ mktemp ++ local LAST_ERR=/tmp/tmp.YiKoe5cWtd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SDCFunQTOH ++ cat /tmp/tmp.YiKoe5cWtd ++ rm /tmp/tmp.SDCFunQTOH /tmp/tmp.YiKoe5cWtd ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V2F76IdQc0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.WGKIkVGVVS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.V2F76IdQc0 ++ cat /tmp/tmp.WGKIkVGVVS ++ rm /tmp/tmp.V2F76IdQc0 /tmp/tmp.WGKIkVGVVS ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.E2GOiMN5vP +++ mktemp ++ local LAST_ERR=/tmp/tmp.S2puHAJYkG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.E2GOiMN5vP ++ cat /tmp/tmp.S2puHAJYkG ++ rm /tmp/tmp.E2GOiMN5vP /tmp/tmp.S2puHAJYkG ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.pROZEA7xwu ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.FF8YQ5Zdjb +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.pROZEA7xwu +++++ cat /tmp/tmp.FF8YQ5Zdjb +++++ rm /tmp/tmp.pROZEA7xwu /tmp/tmp.FF8YQ5Zdjb +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.oZUx2T6IXi ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.0PB4KNf5Rm +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.oZUx2T6IXi +++++ cat /tmp/tmp.0PB4KNf5Rm +++++ rm /tmp/tmp.oZUx2T6IXi /tmp/tmp.0PB4KNf5Rm +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JIuG6qNgaz +++ mktemp ++ local LAST_ERR=/tmp/tmp.AjMmcdYJZy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JIuG6qNgaz ++ cat /tmp/tmp.AjMmcdYJZy ++ rm /tmp/tmp.JIuG6qNgaz /tmp/tmp.AjMmcdYJZy ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lOyk014OFz +++ mktemp ++ local LAST_ERR=/tmp/tmp.4fw9f2TMrF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lOyk014OFz ++ cat /tmp/tmp.4fw9f2TMrF ++ rm /tmp/tmp.lOyk014OFz /tmp/tmp.4fw9f2TMrF ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ubIT7R8ogb +++ mktemp ++ local LAST_ERR=/tmp/tmp.X5o2cTlICG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ubIT7R8ogb ++ cat /tmp/tmp.X5o2cTlICG ++ rm /tmp/tmp.ubIT7R8ogb /tmp/tmp.X5o2cTlICG ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.NFR6Ya3l6j ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.3LqQWP2mzW +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.NFR6Ya3l6j +++++ cat /tmp/tmp.3LqQWP2mzW +++++ rm /tmp/tmp.NFR6Ya3l6j /tmp/tmp.3LqQWP2mzW +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.z3syuKahvw ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.p8tiU7MYtl +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.z3syuKahvw +++++ cat /tmp/tmp.p8tiU7MYtl +++++ rm /tmp/tmp.z3syuKahvw /tmp/tmp.p8tiU7MYtl +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kzFYF9lP4y +++ mktemp ++ local LAST_ERR=/tmp/tmp.fT6PBuVGtH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kzFYF9lP4y ++ cat /tmp/tmp.fT6PBuVGtH ++ rm /tmp/tmp.kzFYF9lP4y /tmp/tmp.fT6PBuVGtH ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sBhKk403M3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.br5rjEHibO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sBhKk403M3 ++ cat /tmp/tmp.br5rjEHibO ++ rm /tmp/tmp.sBhKk403M3 /tmp/tmp.br5rjEHibO ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1Aa4dc9i9G +++ mktemp ++ local LAST_ERR=/tmp/tmp.eU4a5Zh2mb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1Aa4dc9i9G ++ cat /tmp/tmp.eU4a5Zh2mb ++ rm /tmp/tmp.1Aa4dc9i9G /tmp/tmp.eU4a5Zh2mb ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.TUJ7arapC4 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ZlV6EY4Kql +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.TUJ7arapC4 +++++ cat /tmp/tmp.ZlV6EY4Kql +++++ rm /tmp/tmp.TUJ7arapC4 /tmp/tmp.ZlV6EY4Kql +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.zNeOa02EzO ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.zCU4AUXaG0 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.zNeOa02EzO +++++ cat /tmp/tmp.zCU4AUXaG0 +++++ rm /tmp/tmp.zNeOa02EzO /tmp/tmp.zCU4AUXaG0 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Mcpd1YiUWI +++ mktemp ++ local LAST_ERR=/tmp/tmp.9VVfaTRnuH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Mcpd1YiUWI ++ cat /tmp/tmp.9VVfaTRnuH ++ rm /tmp/tmp.Mcpd1YiUWI /tmp/tmp.9VVfaTRnuH ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3HYzWXSpjd +++ mktemp ++ local LAST_ERR=/tmp/tmp.o22AnTGaLn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3HYzWXSpjd ++ cat /tmp/tmp.o22AnTGaLn ++ rm /tmp/tmp.3HYzWXSpjd /tmp/tmp.o22AnTGaLn ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RWyN9kn0U7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RKqIk2kvlY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RWyN9kn0U7 ++ cat /tmp/tmp.RKqIk2kvlY ++ rm /tmp/tmp.RWyN9kn0U7 /tmp/tmp.RKqIk2kvlY ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.j7oEMP5iy6 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.kOls848jlM +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.j7oEMP5iy6 +++++ cat /tmp/tmp.kOls848jlM +++++ rm /tmp/tmp.j7oEMP5iy6 /tmp/tmp.kOls848jlM +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.yGNowvpbgZ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.jYtUy5RWZC +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.yGNowvpbgZ +++++ cat /tmp/tmp.jYtUy5RWZC +++++ rm /tmp/tmp.yGNowvpbgZ /tmp/tmp.jYtUy5RWZC +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.exUgMoGv4i +++ mktemp ++ local LAST_ERR=/tmp/tmp.4EErTa456i ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.exUgMoGv4i ++ cat /tmp/tmp.4EErTa456i ++ rm /tmp/tmp.exUgMoGv4i /tmp/tmp.4EErTa456i ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mfk66MxnsV +++ mktemp ++ local LAST_ERR=/tmp/tmp.nczAJmKJUP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mfk66MxnsV ++ cat /tmp/tmp.nczAJmKJUP ++ rm /tmp/tmp.mfk66MxnsV /tmp/tmp.nczAJmKJUP ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ie9PqkeJH7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.xhQ2fT8vNK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ie9PqkeJH7 ++ cat /tmp/tmp.xhQ2fT8vNK ++ rm /tmp/tmp.Ie9PqkeJH7 /tmp/tmp.xhQ2fT8vNK ++ return 0 + pxc_ready=2 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.TZGTWO2mEd ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.98wa2UcHZK +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.TZGTWO2mEd +++++ cat /tmp/tmp.98wa2UcHZK +++++ rm /tmp/tmp.TZGTWO2mEd /tmp/tmp.98wa2UcHZK +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ILUPZ9jzsJ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.7aSPqVyZRj +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ILUPZ9jzsJ +++++ cat /tmp/tmp.7aSPqVyZRj +++++ rm /tmp/tmp.ILUPZ9jzsJ /tmp/tmp.7aSPqVyZRj +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nqnHsD8j57 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Rq9cWZFG5U ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nqnHsD8j57 ++ cat /tmp/tmp.Rq9cWZFG5U ++ rm /tmp/tmp.nqnHsD8j57 /tmp/tmp.Rq9cWZFG5U ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sYPMPOfV0p +++ mktemp ++ local LAST_ERR=/tmp/tmp.HNqFVXE9a1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sYPMPOfV0p ++ cat /tmp/tmp.HNqFVXE9a1 ++ rm /tmp/tmp.sYPMPOfV0p /tmp/tmp.HNqFVXE9a1 ++ return 0 + state=ready ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f1NFssfr2z +++ mktemp ++ local LAST_ERR=/tmp/tmp.xu5x82syCw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.f1NFssfr2z ++ cat /tmp/tmp.xu5x82syCw ++ rm /tmp/tmp.f1NFssfr2z /tmp/tmp.xu5x82syCw ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.s4GBSfQDin ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Q8jyC8YDJg +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.s4GBSfQDin +++++ cat /tmp/tmp.Q8jyC8YDJg +++++ rm /tmp/tmp.s4GBSfQDin /tmp/tmp.Q8jyC8YDJg +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.WpUN5LiSiP ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.bFJeOO5b4g +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.WpUN5LiSiP +++++ cat /tmp/tmp.bFJeOO5b4g +++++ rm /tmp/tmp.WpUN5LiSiP /tmp/tmp.bFJeOO5b4g +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oHcXCrmqn5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.V2gSxYRB2C ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oHcXCrmqn5 ++ cat /tmp/tmp.V2gSxYRB2C ++ rm /tmp/tmp.oHcXCrmqn5 /tmp/tmp.V2gSxYRB2C ++ return 0 + proxy_ready=3 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 3 == \3 ]] + echo + return + run_mysql 'DELETE FROM myApp.myApp WHERE id=100500' '-h 34.118.239.56 -uroot -proot_password' + local 'command=DELETE FROM myApp.myApp WHERE id=100500' + local 'uri=-h 34.118.239.56 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WsuFXJm5xJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.RqcNkB4zhp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WsuFXJm5xJ ++ cat /tmp/tmp.RqcNkB4zhp ++ rm /tmp/tmp.WsuFXJm5xJ /tmp/tmp.RqcNkB4zhp ++ return 0 + client_pod=pxc-client-56fd5498cd-c947g + wait_pod pxc-client-56fd5498cd-c947g + local pod=pxc-client-56fd5498cd-c947g + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-56fd5498cd-c947g + local container= + set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 03:18:48.586427 5642 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782962326479883000&timeoutSeconds=488&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'DELETE FROM myApp.testSourceReplica WHERE id=100700' '-h 34.118.239.56 -uroot -proot_password' + local 'command=DELETE FROM myApp.testSourceReplica WHERE id=100700' + local 'uri=-h 34.118.239.56 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Qh5sAUPAwW +++ mktemp ++ local LAST_ERR=/tmp/tmp.mGSsCx8FuT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Qh5sAUPAwW ++ cat /tmp/tmp.mGSsCx8FuT ++ rm /tmp/tmp.Qh5sAUPAwW /tmp/tmp.mGSsCx8FuT ++ return 0 + client_pod=pxc-client-56fd5498cd-c947g + wait_pod pxc-client-56fd5498cd-c947g + local pod=pxc-client-56fd5498cd-c947g + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-56fd5498cd-c947g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 03:18:59.739635 7590 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782962338626853000&timeoutSeconds=365&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + desc 'configure old replica as source' + set +o xtrace ----------------------------------------------------------------------------------- configure old replica as source ----------------------------------------------------------------------------------- + kubectl patch pxc cross-site-replica -n cross-site-proxysql-replica-17697 --type=json '-p=[{'\''op'\'': '\''remove'\'', '\''path'\'': '\''/spec/pxc/replicationChannels/0'\''}]' perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + kubectl_bin patch pxc cross-site-replica -n cross-site-proxysql-replica-17697 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": true}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.GQ5oVS4P9Z ++ mktemp + local LAST_ERR=/tmp/tmp.olI6KbXExV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-replica -n cross-site-proxysql-replica-17697 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": true}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GQ5oVS4P9Z perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + cat /tmp/tmp.olI6KbXExV + rm /tmp/tmp.GQ5oVS4P9Z /tmp/tmp.olI6KbXExV + return 0 + desc 'wait for promoted cluster to be ready' + set +o xtrace ----------------------------------------------------------------------------------- wait for promoted cluster to be ready ----------------------------------------------------------------------------------- + switch_namespace cross-site-proxysql-replica-17697 + local ns=cross-site-proxysql-replica-17697 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.NETyyelDmR +++ mktemp ++ local LAST_ERR=/tmp/tmp.S3910WlAI7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NETyyelDmR ++ cat /tmp/tmp.S3910WlAI7 ++ rm /tmp/tmp.NETyyelDmR /tmp/tmp.S3910WlAI7 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-replica-17697 ++ mktemp + local LAST_OUT=/tmp/tmp.DLwpCNtIEk ++ mktemp + local LAST_ERR=/tmp/tmp.d8Po0dKXNh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-replica-17697 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DLwpCNtIEk Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7" modified. + cat /tmp/tmp.d8Po0dKXNh + rm /tmp/tmp.DLwpCNtIEk /tmp/tmp.d8Po0dKXNh + return 0 + wait_cluster_consistency cross-site-replica 3 3 + local cluster_name=cross-site-replica + local pxc_size=3 + local proxy_size=3 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/cross-site-replica to be ready' waiting for pxc/cross-site-replica to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nxQt7RvlGD +++ mktemp ++ local LAST_ERR=/tmp/tmp.j62BQdBXWU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nxQt7RvlGD ++ cat /tmp/tmp.j62BQdBXWU ++ rm /tmp/tmp.nxQt7RvlGD /tmp/tmp.j62BQdBXWU ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xzpL1lZFNV +++ mktemp ++ local LAST_ERR=/tmp/tmp.8GJOZ16DRN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xzpL1lZFNV ++ cat /tmp/tmp.8GJOZ16DRN ++ rm /tmp/tmp.xzpL1lZFNV /tmp/tmp.8GJOZ16DRN ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.HVbCMs0nVB ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.OTOSbKqrgy +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.HVbCMs0nVB +++++ cat /tmp/tmp.OTOSbKqrgy +++++ rm /tmp/tmp.HVbCMs0nVB /tmp/tmp.OTOSbKqrgy +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.1RXVeSiTop ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.K4MJCZOlfb +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.1RXVeSiTop +++++ cat /tmp/tmp.K4MJCZOlfb +++++ rm /tmp/tmp.1RXVeSiTop /tmp/tmp.K4MJCZOlfb +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3g8ONwSBQp +++ mktemp ++ local LAST_ERR=/tmp/tmp.A6AA2PW6Uk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3g8ONwSBQp ++ cat /tmp/tmp.A6AA2PW6Uk ++ rm /tmp/tmp.3g8ONwSBQp /tmp/tmp.A6AA2PW6Uk ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KVnJgROAxY +++ mktemp ++ local LAST_ERR=/tmp/tmp.8HpoyJfB7V ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KVnJgROAxY ++ cat /tmp/tmp.8HpoyJfB7V ++ rm /tmp/tmp.KVnJgROAxY /tmp/tmp.8HpoyJfB7V ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fhfMA3VQ54 +++ mktemp ++ local LAST_ERR=/tmp/tmp.TXquSRVj7o ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fhfMA3VQ54 ++ cat /tmp/tmp.TXquSRVj7o ++ rm /tmp/tmp.fhfMA3VQ54 /tmp/tmp.TXquSRVj7o ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.dg1n91NaU2 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.iGD416oqzB +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.dg1n91NaU2 +++++ cat /tmp/tmp.iGD416oqzB +++++ rm /tmp/tmp.dg1n91NaU2 /tmp/tmp.iGD416oqzB +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.G9F8wlG4d2 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Arnqxv2NzR +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.G9F8wlG4d2 +++++ cat /tmp/tmp.Arnqxv2NzR +++++ rm /tmp/tmp.G9F8wlG4d2 /tmp/tmp.Arnqxv2NzR +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.09K5qJoVlT +++ mktemp ++ local LAST_ERR=/tmp/tmp.BM0HfMr8Qk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.09K5qJoVlT ++ cat /tmp/tmp.BM0HfMr8Qk ++ rm /tmp/tmp.09K5qJoVlT /tmp/tmp.BM0HfMr8Qk ++ return 0 + proxy_ready=2 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RQIdtkObVB +++ mktemp ++ local LAST_ERR=/tmp/tmp.8XPS2rzXIO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RQIdtkObVB ++ cat /tmp/tmp.8XPS2rzXIO ++ rm /tmp/tmp.RQIdtkObVB /tmp/tmp.8XPS2rzXIO ++ return 0 + state=initializing ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ru90F6Fz3s +++ mktemp ++ local LAST_ERR=/tmp/tmp.ur4Qyw0qeV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ru90F6Fz3s ++ cat /tmp/tmp.ur4Qyw0qeV ++ rm /tmp/tmp.Ru90F6Fz3s /tmp/tmp.ur4Qyw0qeV ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.0gP1ShrUkx ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.1Cd29OjHxV +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.0gP1ShrUkx +++++ cat /tmp/tmp.1Cd29OjHxV +++++ rm /tmp/tmp.0gP1ShrUkx /tmp/tmp.1Cd29OjHxV +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Xgi8e2IYti ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.zfW6GCNar9 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Xgi8e2IYti +++++ cat /tmp/tmp.zfW6GCNar9 +++++ rm /tmp/tmp.Xgi8e2IYti /tmp/tmp.zfW6GCNar9 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nq3Mz47Xj2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7j3nR1oja6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nq3Mz47Xj2 ++ cat /tmp/tmp.7j3nR1oja6 ++ rm /tmp/tmp.nq3Mz47Xj2 /tmp/tmp.7j3nR1oja6 ++ return 0 + proxy_ready=3 + [[ initializing == \r\e\a\d\y ]] + echo -n . .+ sleep 5 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kE8jOiITRk +++ mktemp ++ local LAST_ERR=/tmp/tmp.ucWQ73fBXO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kE8jOiITRk ++ cat /tmp/tmp.ucWQ73fBXO ++ rm /tmp/tmp.kE8jOiITRk /tmp/tmp.ucWQ73fBXO ++ return 0 + state=ready ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9uhPonDKBt +++ mktemp ++ local LAST_ERR=/tmp/tmp.U4dcBc1OOt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9uhPonDKBt ++ cat /tmp/tmp.U4dcBc1OOt ++ rm /tmp/tmp.9uhPonDKBt /tmp/tmp.U4dcBc1OOt ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 3 -gt 0 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Fpd4uWd8me ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.yd8vyc56k6 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Fpd4uWd8me +++++ cat /tmp/tmp.yd8vyc56k6 +++++ rm /tmp/tmp.Fpd4uWd8me /tmp/tmp.yd8vyc56k6 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.Yg9JeeuVto ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Tk4Ixedpux +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.Yg9JeeuVto +++++ cat /tmp/tmp.Tk4Ixedpux +++++ rm /tmp/tmp.Yg9JeeuVto /tmp/tmp.Tk4Ixedpux +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nRn9iMBsMM +++ mktemp ++ local LAST_ERR=/tmp/tmp.BSyglGxtgl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nRn9iMBsMM ++ cat /tmp/tmp.BSyglGxtgl ++ rm /tmp/tmp.nRn9iMBsMM /tmp/tmp.BSyglGxtgl ++ return 0 + proxy_ready=3 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 3 == \3 ]] + echo + return + desc 'check that ProxySQL on promoted cluster has writer (11) and reader (10) hostgroups' + set +o xtrace ----------------------------------------------------------------------------------- check that ProxySQL on promoted cluster has writer (11) and reader (10) hostgroups ----------------------------------------------------------------------------------- + sleep 60 + check_proxysql_hostgroup cross-site-replica 3 11 populated + local cluster=cross-site-replica + local size=3 + local hostgroup_id=11 + local expected=populated + local 'admin_uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' + local last_pod=2 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + local count ++ run_mysql_local 'SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-replica-proxysql-0 proxysql ++ local 'command=SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=cross-site-replica-proxysql-0 ++ local container_name=proxysql ++ set +o xtrace + count=1 + [[ populated == \e\m\p\t\y ]] + [[ populated == \p\o\p\u\l\a\t\e\d ]] + [[ 1 -eq 0 ]] + for i in '$(seq 0 $last_pod)' + local count ++ run_mysql_local 'SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-replica-proxysql-1 proxysql ++ local 'command=SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=cross-site-replica-proxysql-1 ++ local container_name=proxysql ++ set +o xtrace + count=1 + [[ populated == \e\m\p\t\y ]] + [[ populated == \p\o\p\u\l\a\t\e\d ]] + [[ 1 -eq 0 ]] + for i in '$(seq 0 $last_pod)' + local count ++ run_mysql_local 'SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-replica-proxysql-2 proxysql ++ local 'command=SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status='\''ONLINE'\'';' ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=cross-site-replica-proxysql-2 ++ local container_name=proxysql ++ set +o xtrace + count=1 + [[ populated == \e\m\p\t\y ]] + [[ populated == \p\o\p\u\l\a\t\e\d ]] + [[ 1 -eq 0 ]] + check_proxysql_hostgroup cross-site-replica 3 10 populated + local cluster=cross-site-replica + local size=3 + local hostgroup_id=10 + local expected=populated + local 'admin_uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' + local last_pod=2 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + local count ++ run_mysql_local 'SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=10 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-replica-proxysql-0 proxysql ++ local 'command=SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=10 AND status='\''ONLINE'\'';' ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=cross-site-replica-proxysql-0 ++ local container_name=proxysql ++ set +o xtrace + count=3 + [[ populated == \e\m\p\t\y ]] + [[ populated == \p\o\p\u\l\a\t\e\d ]] + [[ 3 -eq 0 ]] + for i in '$(seq 0 $last_pod)' + local count ++ run_mysql_local 'SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=10 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-replica-proxysql-1 proxysql ++ local 'command=SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=10 AND status='\''ONLINE'\'';' ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=cross-site-replica-proxysql-1 ++ local container_name=proxysql ++ set +o xtrace + count=3 + [[ populated == \e\m\p\t\y ]] + [[ populated == \p\o\p\u\l\a\t\e\d ]] + [[ 3 -eq 0 ]] + for i in '$(seq 0 $last_pod)' + local count ++ run_mysql_local 'SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=10 AND status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-replica-proxysql-2 proxysql ++ local 'command=SELECT COUNT(*) FROM runtime_mysql_servers WHERE hostgroup_id=10 AND status='\''ONLINE'\'';' ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=cross-site-replica-proxysql-2 ++ local container_name=proxysql ++ set +o xtrace + count=3 + [[ populated == \e\m\p\t\y ]] + [[ populated == \p\o\p\u\l\a\t\e\d ]] + [[ 3 -eq 0 ]] + desc 'write data via ProxySQL on promoted cluster and verify' + set +o xtrace ----------------------------------------------------------------------------------- write data via ProxySQL on promoted cluster and verify ----------------------------------------------------------------------------------- + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testPromotion (id int PRIMARY KEY);' '-h cross-site-replica-proxysql -p3306 -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testPromotion (id int PRIMARY KEY);' + local 'uri=-h cross-site-replica-proxysql -p3306 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ba3CkxQmtv +++ mktemp ++ local LAST_ERR=/tmp/tmp.46wqKp0syx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ba3CkxQmtv ++ cat /tmp/tmp.46wqKp0syx ++ rm /tmp/tmp.Ba3CkxQmtv /tmp/tmp.46wqKp0syx ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-56fd5498cd-f2ffc + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:21:25.322381 28920 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782962482840210000&timeoutSeconds=348&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.testPromotion (id) VALUES (100900)' '-h cross-site-replica-proxysql -p3306 -uroot -proot_password' + local 'command=INSERT myApp.testPromotion (id) VALUES (100900)' + local 'uri=-h cross-site-replica-proxysql -p3306 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RpOBjsYF5j +++ mktemp ++ local LAST_ERR=/tmp/tmp.7fgLrNBSV2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RpOBjsYF5j ++ cat /tmp/tmp.7fgLrNBSV2 ++ rm /tmp/tmp.RpOBjsYF5j /tmp/tmp.7fgLrNBSV2 ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-f2ffc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:21:38.537314 30906 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782962495541008000&timeoutSeconds=400&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + compare_mysql_cmd select-4 'SELECT * from myApp.testPromotion;' '-h cross-site-replica-proxysql -p3306 -uroot -proot_password' + local command_id=select-4 + local 'command=SELECT * from myApp.testPromotion;' + local 'uri=-h cross-site-replica-proxysql -p3306 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-4.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-4-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.testPromotion;' '-h cross-site-replica-proxysql -p3306 -uroot -proot_password' + local 'command=SELECT * from myApp.testPromotion;' + local 'uri=-h cross-site-replica-proxysql -p3306 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GjNxTkg02l +++ mktemp ++ local LAST_ERR=/tmp/tmp.43Jodnq414 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GjNxTkg02l ++ cat /tmp/tmp.43Jodnq414 ++ rm /tmp/tmp.GjNxTkg02l /tmp/tmp.43Jodnq414 ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-f2ffc ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:21:54.761354 813 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782962512332130000&timeoutSeconds=397&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-4.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-4.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-4.sql /tmp/tmp.xyHcWb7plX/select-4.sql + switch_namespace cross-site-proxysql-28035 + local ns=cross-site-proxysql-28035 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.QSRakenF2R +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ui6JiDJOUC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QSRakenF2R ++ cat /tmp/tmp.Ui6JiDJOUC ++ rm /tmp/tmp.QSRakenF2R /tmp/tmp.Ui6JiDJOUC ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-28035 ++ mktemp + local LAST_OUT=/tmp/tmp.Lb5EfZVWhM ++ mktemp + local LAST_ERR=/tmp/tmp.0AyE9i4YoU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-28035 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Lb5EfZVWhM Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7" modified. + cat /tmp/tmp.0AyE9i4YoU + rm /tmp/tmp.Lb5EfZVWhM /tmp/tmp.0AyE9i4YoU + return 0 + desc 'configure old source as replica' + set +o xtrace ----------------------------------------------------------------------------------- configure old source as replica ----------------------------------------------------------------------------------- + kubectl patch pxc cross-site-source -n cross-site-proxysql-28035 --type=json '-p=[{'\''op'\'': '\''remove'\'', '\''path'\'': '\''/spec/pxc/replicationChannels/0'\''}]' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + kubectl_bin patch pxc cross-site-source -n cross-site-proxysql-28035 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "34.118.239.36", "port": 3306, "weight": 100},{"host": "34.118.228.40", "port": 3306, "weight": 100},{"host": "34.118.232.120", "port": 3306, "weight": 100}]}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.DJ3dD2OXGf ++ mktemp + local LAST_ERR=/tmp/tmp.OnKV2jxoZK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-source -n cross-site-proxysql-28035 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "34.118.239.36", "port": 3306, "weight": 100},{"host": "34.118.228.40", "port": 3306, "weight": 100},{"host": "34.118.232.120", "port": 3306, "weight": 100}]}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DJ3dD2OXGf perconaxtradbcluster.pxc.percona.com/cross-site-source patched + cat /tmp/tmp.OnKV2jxoZK + rm /tmp/tmp.DJ3dD2OXGf /tmp/tmp.OnKV2jxoZK + return 0 + switch_namespace cross-site-proxysql-replica-17697 + local ns=cross-site-proxysql-replica-17697 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.0GWIEAWljj +++ mktemp ++ local LAST_ERR=/tmp/tmp.e4Ot9iFAqu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0GWIEAWljj ++ cat /tmp/tmp.e4Ot9iFAqu ++ rm /tmp/tmp.0GWIEAWljj /tmp/tmp.e4Ot9iFAqu ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-replica-17697 ++ mktemp + local LAST_OUT=/tmp/tmp.YPOCec4p0k ++ mktemp + local LAST_ERR=/tmp/tmp.M2ZruRbpHz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-replica-17697 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YPOCec4p0k Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7" modified. + cat /tmp/tmp.M2ZruRbpHz + rm /tmp/tmp.YPOCec4p0k /tmp/tmp.M2ZruRbpHz + return 0 + desc 'Write data to replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- Write data to replica cluster ----------------------------------------------------------------------------------- + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testReplicaSource (id int PRIMARY KEY);' '-h 34.118.239.36 -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testReplicaSource (id int PRIMARY KEY);' + local 'uri=-h 34.118.239.36 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jgRzObW74l +++ mktemp ++ local LAST_ERR=/tmp/tmp.LFuhYqqJYO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jgRzObW74l ++ cat /tmp/tmp.LFuhYqqJYO ++ rm /tmp/tmp.jgRzObW74l /tmp/tmp.LFuhYqqJYO ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-56fd5498cd-f2ffc + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:22:20.045085 4415 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782962539157759002&timeoutSeconds=423&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.testReplicaSource (id) VALUES (100800)' '-h 34.118.239.36 -uroot -proot_password' + local 'command=INSERT myApp.testReplicaSource (id) VALUES (100800)' + local 'uri=-h 34.118.239.36 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H0yAMJAtOy +++ mktemp ++ local LAST_ERR=/tmp/tmp.NmgBstEMv1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.H0yAMJAtOy ++ cat /tmp/tmp.NmgBstEMv1 ++ rm /tmp/tmp.H0yAMJAtOy /tmp/tmp.NmgBstEMv1 ++ return 0 + client_pod=pxc-client-56fd5498cd-f2ffc + wait_pod pxc-client-56fd5498cd-f2ffc + local pod=pxc-client-56fd5498cd-f2ffc + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-56fd5498cd-f2ffc ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-f2ffc condition met E0702 03:22:36.992222 6990 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-replica-17697/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-f2ffc&resourceVersion=1782962556521151011&timeoutSeconds=444&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-f2ffc to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + wait_replication_row 34.118.239.56 'SELECT * FROM myApp.testReplicaSource WHERE id=100800' + local host=34.118.239.56 + local 'query=SELECT * FROM myApp.testReplicaSource WHERE id=100800' + local retry=0 + grep -q . + run_mysql 'SELECT * FROM myApp.testReplicaSource WHERE id=100800' '-h 34.118.239.56 -uroot -proot_password' + desc 'Check replication works between replica -> source' + set +o xtrace ----------------------------------------------------------------------------------- Check replication works between replica -> source ----------------------------------------------------------------------------------- + switch_namespace cross-site-proxysql-28035 + local ns=cross-site-proxysql-28035 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.p87xlcQiSL +++ mktemp ++ local LAST_ERR=/tmp/tmp.ocPwaorwZ6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.p87xlcQiSL ++ cat /tmp/tmp.ocPwaorwZ6 ++ rm /tmp/tmp.p87xlcQiSL /tmp/tmp.ocPwaorwZ6 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-28035 ++ mktemp + local LAST_OUT=/tmp/tmp.gHd9CAdhfS ++ mktemp + local LAST_ERR=/tmp/tmp.RiznKwVJVi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7 --namespace=cross-site-proxysql-28035 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gHd9CAdhfS Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster7" modified. + cat /tmp/tmp.RiznKwVJVi + rm /tmp/tmp.gHd9CAdhfS /tmp/tmp.RiznKwVJVi + return 0 + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 34.118.239.56 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 34.118.239.56 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-3-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 34.118.239.56 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 34.118.239.56 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tgzOIUPsws +++ mktemp ++ local LAST_ERR=/tmp/tmp.6AmdOwBz54 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tgzOIUPsws ++ cat /tmp/tmp.6AmdOwBz54 ++ rm /tmp/tmp.tgzOIUPsws /tmp/tmp.6AmdOwBz54 ++ return 0 + client_pod=pxc-client-56fd5498cd-c947g + wait_pod pxc-client-56fd5498cd-c947g + local pod=pxc-client-56fd5498cd-c947g + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-56fd5498cd-c947g + local container= + set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 03:23:08.244387 12067 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782962587215556000&timeoutSeconds=304&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-3.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-3.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-3.sql /tmp/tmp.xyHcWb7plX/select-3.sql + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 34.118.235.139 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 34.118.235.139 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-3-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 34.118.235.139 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 34.118.235.139 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IXldDWZgEU +++ mktemp ++ local LAST_ERR=/tmp/tmp.xYP4nemHSx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IXldDWZgEU ++ cat /tmp/tmp.xYP4nemHSx ++ rm /tmp/tmp.IXldDWZgEU /tmp/tmp.xYP4nemHSx ++ return 0 + client_pod=pxc-client-56fd5498cd-c947g + wait_pod pxc-client-56fd5498cd-c947g + local pod=pxc-client-56fd5498cd-c947g + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-c947g ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 03:23:23.635857 14182 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782962601399612000&timeoutSeconds=508&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-3.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-3.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-3.sql /tmp/tmp.xyHcWb7plX/select-3.sql + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 34.118.228.242 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 34.118.228.242 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.4 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-3-84.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 8\.0 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.4 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 34.118.228.242 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 34.118.228.242 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CA22TxuWyq +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ds1gkSf4uM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CA22TxuWyq ++ cat /tmp/tmp.Ds1gkSf4uM ++ rm /tmp/tmp.CA22TxuWyq /tmp/tmp.Ds1gkSf4uM ++ return 0 + client_pod=pxc-client-56fd5498cd-c947g + wait_pod pxc-client-56fd5498cd-c947g + local pod=pxc-client-56fd5498cd-c947g + local max_retry=480 + local ns= ++ echo pxc-client-56fd5498cd-c947g ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-56fd5498cd-c947g condition met E0702 03:23:41.686801 16733 reflector.go:227] "Failed to watch" err="Get \"https://34.134.74.193/api/v1/namespaces/cross-site-proxysql-28035/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-56fd5498cd-c947g&resourceVersion=1782962618181111000&timeoutSeconds=411&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-56fd5498cd-c947g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.xyHcWb7plX/select-3.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.xyHcWb7plX/select-3.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/cross-site-proxysql/compare/select-3.sql /tmp/tmp.xyHcWb7plX/select-3.sql + destroy cross-site-proxysql-28035 true + local namespace=cross-site-proxysql-28035 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v level=info + grep -v 'the object has been modified' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'get backup status: Job.batch' + tee /tmp/tmp.xyHcWb7plX/operator.log + sort -u ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.jTygsm0jx4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.un5s3Pda4E ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jTygsm0jx4 ++ cat /tmp/tmp.un5s3Pda4E ++ rm /tmp/tmp.jTygsm0jx4 /tmp/tmp.un5s3Pda4E ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-65b5d85575-9dbzk ++ mktemp + local LAST_OUT=/tmp/tmp.1Oy08GXonN ++ mktemp + local LAST_ERR=/tmp/tmp.alqBpsAoqQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-65b5d85575-9dbzk + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1Oy08GXonN + cat /tmp/tmp.alqBpsAoqQ + rm /tmp/tmp.1Oy08GXonN /tmp/tmp.alqBpsAoqQ + return 0 2026-07-02T02:40:38.377Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-07-02T02:40:38.377Z INFO setup Manager starting up {"gitCommit": "dcdf1e8aedba334999b2bd5fa9efcdb1f10e79e0", "gitBranch": "PR-2539-dcdf1e8a", "buildTime": "2026-07-02T01:14:56Z", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} 2026-07-02T02:40:38.377Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.33.12-gke.1059000"} 2026-07-02T02:40:38.381Z INFO setup Registering Components. 2026-07-02T02:40:39.175Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-07-02T02:40:39.175Z INFO controller-runtime.metrics Starting metrics server 2026-07-02T02:40:39.175Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-07-02T02:40:39.175Z INFO controller-runtime.webhook Starting webhook server 2026-07-02T02:40:39.175Z INFO setup Starting the Cmd. 2026-07-02T02:40:39.175Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-07-02T02:40:39.176Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-07-02T02:40:39.176Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-07-02T02:40:39.176Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-07-02T02:40:39.277Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-07-02T02:40:39.318Z DEBUG events percona-xtradb-cluster-operator-65b5d85575-9dbzk_27e2e4b2-59df-423c-a5ef-59ab838c100d became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"88fffc88-a644-4e46-9d3d-e0f89d6a70b1","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1782960039299343009"}, "reason": "LeaderElection"} 2026-07-02T02:40:39.318Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-07-02T02:40:39.318Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-07-02T02:40:39.318Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-07-02T02:40:39.318Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-07-02T02:40:39.319Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-07-02T02:40:39.419Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-07-02T02:40:39.419Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-07-02T02:40:39.519Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-07-02T02:40:39.519Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-07-02T02:40:39.519Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-07-02T02:40:39.519Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-07-02T02:43:16.808Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1e57ee5f-43f2-4e79-8781-116493351ec0", "version": "1.21.0"} 2026-07-02T02:43:18.476Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1e57ee5f-43f2-4e79-8781-116493351ec0", "object": "auto-cross-site-source-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T02:43:18.595Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1e57ee5f-43f2-4e79-8781-116493351ec0", "object": "cross-site-source-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T02:43:18.642Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1e57ee5f-43f2-4e79-8781-116493351ec0", "object": "cross-site-source-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T02:43:18.700Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1e57ee5f-43f2-4e79-8781-116493351ec0", "object": "cross-site-source-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:43:18.769Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1e57ee5f-43f2-4e79-8781-116493351ec0", "object": "cross-site-source-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:43:18.833Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1e57ee5f-43f2-4e79-8781-116493351ec0", "object": "cross-site-source-pxc-0", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:43:18.917Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1e57ee5f-43f2-4e79-8781-116493351ec0", "object": "cross-site-source-pxc-1", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:43:19.097Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1e57ee5f-43f2-4e79-8781-116493351ec0", "object": "cross-site-source-pxc-2", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:43:19.263Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1e57ee5f-43f2-4e79-8781-116493351ec0", "object": "cross-site-source-proxysql", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:43:19.399Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1e57ee5f-43f2-4e79-8781-116493351ec0", "object": "cross-site-source-proxysql-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:43:20.394Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "6f54b719-c98e-4d12-9ca3-799fbc3f14e6", "object": "cross-site-source-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T02:43:20.427Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "6f54b719-c98e-4d12-9ca3-799fbc3f14e6", "object": "cross-site-source-proxysql", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T02:44:37.309Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3243e560-477f-4459-9d23-7a7bcefb7d9b", "user": "operator"} 2026-07-02T02:44:37.360Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3243e560-477f-4459-9d23-7a7bcefb7d9b", "user": "monitor"} 2026-07-02T02:44:37.413Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3243e560-477f-4459-9d23-7a7bcefb7d9b"} 2026-07-02T02:44:37.444Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3243e560-477f-4459-9d23-7a7bcefb7d9b"} 2026-07-02T02:44:37.483Z INFO monitor user audit admin privilege granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3243e560-477f-4459-9d23-7a7bcefb7d9b"} 2026-07-02T02:44:37.519Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3243e560-477f-4459-9d23-7a7bcefb7d9b", "user": "xtrabackup"} 2026-07-02T02:44:37.559Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3243e560-477f-4459-9d23-7a7bcefb7d9b"} 2026-07-02T02:44:37.591Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3243e560-477f-4459-9d23-7a7bcefb7d9b", "user": "replication"} 2026-07-02T02:44:37.601Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3243e560-477f-4459-9d23-7a7bcefb7d9b", "err": "get primary pxc pod: not found"} 2026-07-02T02:44:42.725Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "16e63dd0-e064-4a1a-b399-0eaf1255e123", "err": "get primary pxc pod: not found"} 2026-07-02T02:44:47.842Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "308bcbd4-13b4-4f8d-b61f-1d38e7be9e19", "err": "get primary pxc pod: not found"} 2026-07-02T02:47:13.922Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "cf984f52-acb9-4875-b04f-13aaf1d78bc5", "user": "root"} 2026-07-02T02:47:13.996Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "cf984f52-acb9-4875-b04f-13aaf1d78bc5", "new version": "8.4.7-7.1"} 2026-07-02T02:47:17.736Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "cf984f52-acb9-4875-b04f-13aaf1d78bc5"} 2026-07-02T02:47:23.945Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d5c76f36-0ed0-4efe-a0e4-658ac73ad8a6"} 2026-07-02T02:47:29.472Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "48ba9bac-c921-44ae-be5c-b7648bd17c86"} 2026-07-02T02:47:34.875Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "5ca1f646-bca2-4522-a2b6-56a51c4a85d5"} 2026-07-02T02:47:39.756Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3972b846-aecd-4a13-82b9-59bb10261ed0"} 2026-07-02T02:47:44.672Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ec30b9a1-0703-47e3-ab95-39462fa86a06"} 2026-07-02T02:47:50.501Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "b57df2e7-ac1c-4218-96e9-3f33ff9af8d6"} 2026-07-02T02:47:55.539Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "fc963399-5edd-441f-a0f9-878635340aa5"} 2026-07-02T02:48:00.850Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "4272ba21-e996-4d2d-b76f-92ed828a1a8d"} 2026-07-02T02:48:06.460Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "76dd5e54-059e-40b0-884c-ec83eebfadce"} 2026-07-02T02:48:11.646Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "2cc6be2a-5a3e-4533-bcea-1e654bbf74a2"} 2026-07-02T02:48:17.080Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1b197163-f5ad-410a-8abd-2fb1f02801d3"} 2026-07-02T02:48:22.149Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7c294b3e-77e8-42ee-a602-f923dad5b184"} 2026-07-02T02:48:27.478Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "30d7ae11-488c-489d-accc-76e1d66e8d99"} 2026-07-02T02:48:32.644Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "b884d96c-d7a4-4b7b-81e1-812dcceab90f"} 2026-07-02T02:48:37.678Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "4408101a-b8db-4445-9f49-1c7e28dd5be5"} 2026-07-02T02:48:43.174Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "06ca7281-d931-49d1-b23f-0354041d0b50"} 2026-07-02T02:48:48.045Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "84e35ad4-aeda-446e-9dd3-647fee3929d3"} 2026-07-02T02:48:54.023Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "f45d1dce-cadc-474e-83cb-6f8a0b095497"} 2026-07-02T02:48:59.052Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ef67e3af-18d7-4005-9685-43cd01e72b51"} 2026-07-02T02:49:04.459Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e4a0910d-1570-475b-888a-61ccde6b0732"} 2026-07-02T02:49:09.367Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ce7260ee-dcfa-4d68-b5b5-b9d68f6b4605"} 2026-07-02T02:49:14.767Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "37ddef5e-2631-4a0c-bc07-47ed1a13959a"} 2026-07-02T02:49:20.270Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "774d5557-59b0-4a96-a501-265363d66890"} 2026-07-02T02:49:25.046Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "182baa21-44ab-446c-a252-17f52eb04cae"} 2026-07-02T02:49:30.673Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "2868444b-3b05-46e5-b4f9-cf22c8a41821"} 2026-07-02T02:49:36.355Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "c203b7e5-7438-4423-a8f2-0472b5276c0e"} 2026-07-02T02:49:41.438Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "aa7e0706-3a01-46d0-8f26-d89be2d7fa7c"} 2026-07-02T02:49:46.455Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "02291123-1b22-412f-9ea0-71c81e9e0af0"} 2026-07-02T02:49:51.550Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d1adf027-dcce-4715-bdff-14ac16bdeff9"} 2026-07-02T02:49:57.354Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d5aad61d-b4e6-4d43-b4e5-124e3d289bbe"} 2026-07-02T02:50:02.550Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "bcf26392-ed82-4957-b490-0283a1cf38d4"} 2026-07-02T02:50:07.554Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "57354af6-dd59-45f2-9f5e-16b5d2fd4aaa"} 2026-07-02T02:50:13.177Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "2e0e90f1-bade-4f2f-addd-ee2264380358"} 2026-07-02T02:50:18.527Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "9e63ed1a-2c16-4417-ad50-7d0da890efd3"} 2026-07-02T02:50:23.867Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "35e89a90-551b-40f9-a864-92dc04985a8d"} 2026-07-02T02:50:29.440Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "5c31dd26-9f22-4d2c-9fc7-01d259b03282"} 2026-07-02T02:50:34.866Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d9a766a5-c759-42b6-bb79-d87f31f1a7de"} 2026-07-02T02:50:35.292Z INFO Password changed, updating user {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d89dcb11-6ac6-46a0-948d-29d2506b3a47", "user": "replication"} 2026-07-02T02:50:35.306Z INFO Password updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d89dcb11-6ac6-46a0-948d-29d2506b3a47", "user": "replication"} 2026-07-02T02:50:35.417Z INFO MySQL init secret created {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d89dcb11-6ac6-46a0-948d-29d2506b3a47", "secret": "cross-site-source-mysql-init", "user": "replication"} 2026-07-02T02:50:35.470Z INFO Internal secrets updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d89dcb11-6ac6-46a0-948d-29d2506b3a47", "user": "replication"} 2026-07-02T02:50:35.480Z INFO Old password discarded {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d89dcb11-6ac6-46a0-948d-29d2506b3a47", "user": "replication"} 2026-07-02T02:50:39.585Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d89dcb11-6ac6-46a0-948d-29d2506b3a47"} 2026-07-02T02:50:44.245Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "16914d82-cd0e-4995-8e15-6a9d8cc99ab1"} 2026-07-02T02:50:49.741Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "63f0c96d-abd1-48b1-a8d2-c4c55e3d6efc"} 2026-07-02T02:50:55.248Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "4b82910f-10c5-4a55-b283-94f9cd9d7132"} 2026-07-02T02:51:00.041Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "42a0f784-9bbb-46e9-8040-839323e0ccb1"} 2026-07-02T02:51:05.267Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "db40d4c5-feed-4425-924e-b5a220c19545"} 2026-07-02T02:51:10.653Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "eee8520f-98aa-42b0-aba9-3c9bd82467bc"} 2026-07-02T02:51:15.941Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "2546eb89-caf5-429b-af0d-d8fe335817b6"} 2026-07-02T02:51:21.447Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "2b3daaa2-9a77-4f5c-8415-cf0ffdbdaa05"} 2026-07-02T02:51:26.577Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "53432538-3ef2-49ab-826e-6c6cbfa71b66"} 2026-07-02T02:51:26.754Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio-source", "reconcileID": "f42738a6-0602-4d41-8ba9-52ce50dbf278", "cluster": "cross-site-source", "storage": "minio", "allowed": true} 2026-07-02T02:51:26.797Z INFO Created a new backup job {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio-source", "reconcileID": "f42738a6-0602-4d41-8ba9-52ce50dbf278", "namespace": "cross-site-proxysql-28035", "name": "xb-backup-minio-source"} 2026-07-02T02:51:31.750Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ffcc623e-45b3-4732-bd00-c119c5a22097"} 2026-07-02T02:51:31.798Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio-source", "reconcileID": "2ab270e4-32a1-4f02-bcc4-d1c4977b43da", "cluster": "cross-site-source", "storage": "minio", "allowed": true} 2026-07-02T02:51:31.867Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio-source", "reconcileID": "cb4096af-639e-44c5-8e88-cae0ed60f62a", "cluster": "cross-site-source", "storage": "minio", "allowed": true} 2026-07-02T02:51:36.914Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio-source", "reconcileID": "9d030228-0f69-4511-9992-52ccbe3a1ffa", "cluster": "cross-site-source", "storage": "minio", "allowed": true} 2026-07-02T02:51:37.527Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "6fddc7bc-1be7-4e88-8a17-818f72b26ef1"} 2026-07-02T02:51:41.946Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio-source", "reconcileID": "c8249eab-cff7-4d99-b71a-79baa46a31a7", "cluster": "cross-site-source", "storage": "minio", "allowed": true} 2026-07-02T02:51:42.146Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3b1a55bc-2d82-4695-874d-dfe7edfd0a33"} 2026-07-02T02:51:46.994Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio-source", "reconcileID": "acd13710-8b8c-4570-98a4-069e2c83cf7d", "cluster": "cross-site-source", "storage": "minio", "allowed": true} 2026-07-02T02:51:47.477Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "bf119487-ea12-4091-8ef0-98f5d26ae8a4"} 2026-07-02T02:51:52.039Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio-source", "reconcileID": "cb2046f4-0390-4e1e-b699-9a8bef79a202", "cluster": "cross-site-source", "storage": "minio", "allowed": true} 2026-07-02T02:51:53.053Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "fec0f067-1879-45aa-ae04-9cca0835140e"} 2026-07-02T02:51:57.128Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio-source", "reconcileID": "b4fe1bd9-265a-4c6a-9501-7dd631ab02c6", "cluster": "cross-site-source", "storage": "minio", "allowed": true} 2026-07-02T02:51:57.191Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio-source", "reconcileID": "c4486e46-65dd-4ac0-ad20-be5b7568f93e", "cluster": "cross-site-source", "storage": "minio", "allowed": true} 2026-07-02T02:51:57.233Z DEBUG Removing mysql-init secret {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio-source", "reconcileID": "c4486e46-65dd-4ac0-ad20-be5b7568f93e", "job": "xb-backup-minio-source", "secret": "cross-site-source-mysql-init"} 2026-07-02T02:51:57.233Z INFO Backup succeeded {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio-source", "reconcileID": "c4486e46-65dd-4ac0-ad20-be5b7568f93e", "job": "xb-backup-minio-source"} 2026-07-02T02:51:58.146Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1652f887-5a20-4f58-9260-5390e561f6eb"} 2026-07-02T02:52:03.361Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "6aa44600-007b-4af5-ba9c-a3d2993cb41b"} 2026-07-02T02:52:09.164Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "8cb6f56d-d69d-4d1c-bfe3-e2fd927192da"} 2026-07-02T02:52:14.257Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e29bbb6a-e820-44c2-83d6-f4275a8d6e6e"} 2026-07-02T02:52:19.821Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "41081c05-bd13-439a-a5ca-0c81f740fb05"} 2026-07-02T02:52:25.033Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "30a6c279-6a61-454b-bf02-2af80975c1cb"} 2026-07-02T02:52:30.046Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "bf31f743-73a2-413e-8a8e-59503e764aea"} 2026-07-02T02:52:35.389Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "c212aaa3-c55a-4694-b842-20f0e1f48d2b"} 2026-07-02T02:52:40.672Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e884a58a-1561-4db0-9e6f-efb6ee5d4a3a"} 2026-07-02T02:52:46.031Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "5fa34de3-7a4b-40fd-be71-fab7ccb7cd68"} 2026-07-02T02:52:51.579Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "44c64b43-ccd2-42e8-a30f-dfa259ea4c7f"} 2026-07-02T02:52:56.463Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ea828de6-065e-4d8c-8ca4-96f5d7435c45"} 2026-07-02T02:53:01.543Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d220098a-ffbf-4c37-bfee-7c612bb2fcc7"} 2026-07-02T02:53:07.448Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "b0ed887b-16c2-42ee-b089-54bad4c0648f"} 2026-07-02T02:53:11.851Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7261c2e3-a148-42b0-8eed-aa1f5fc26c0f"} 2026-07-02T02:53:17.578Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "29081710-4c29-418e-ba0b-669c9b70d47b"} 2026-07-02T02:53:22.851Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "74539c14-bc55-4620-b01f-693e153a272d"} 2026-07-02T02:53:28.144Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "09a64f0d-d2eb-46d6-b60b-9a5f0c2b3c3a"} 2026-07-02T02:53:33.377Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e415a953-96cf-42f9-be75-2d983d93ee1c"} 2026-07-02T02:53:39.045Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d9b04015-1e5e-4252-b60d-4e1c58a8ae47"} 2026-07-02T02:53:44.121Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "f9020f22-4b2b-4903-86c5-84c5999e6902"} 2026-07-02T02:53:45.500Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c50190fb-487c-4daf-be00-acb4761e8510", "version": "1.21.0"} 2026-07-02T02:53:46.902Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c50190fb-487c-4daf-be00-acb4761e8510", "object": "auto-cross-site-replica-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T02:53:46.920Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c50190fb-487c-4daf-be00-acb4761e8510", "object": "auto-cross-site-replica-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"} 2026-07-02T02:53:47.484Z ERROR Reconciler error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c50190fb-487c-4daf-be00-acb4761e8510", "error": "failed to deploy proxysql: updatePod for proxysql: reconcile config: reconcile autotune config: create or update configmap: configmaps \"auto-cross-site-replica-pxc\" already exists", "errorVerbose": "configmaps \"auto-cross-site-replica-pxc\" already exists\ncreate or update configmap\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileAutotuneConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:101\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:26\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:585\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile autotune config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:28\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:57\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:585\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:59\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:567\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:585\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nupdatePod for proxysql\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:568\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:585\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nfailed to deploy proxysql\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:586\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:392\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T02:53:47.597Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ee8f8470-830e-48ff-a727-0c31c27777bc", "object": "cross-site-replica-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T02:53:47.646Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ee8f8470-830e-48ff-a727-0c31c27777bc", "object": "cross-site-replica-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T02:53:47.708Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ee8f8470-830e-48ff-a727-0c31c27777bc", "object": "cross-site-replica-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:53:47.750Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ee8f8470-830e-48ff-a727-0c31c27777bc", "object": "cross-site-replica-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:53:47.818Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ee8f8470-830e-48ff-a727-0c31c27777bc", "object": "cross-site-replica-pxc-0", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:53:47.977Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ee8f8470-830e-48ff-a727-0c31c27777bc", "object": "cross-site-replica-pxc-1", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:53:48.114Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ee8f8470-830e-48ff-a727-0c31c27777bc", "object": "cross-site-replica-pxc-2", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:53:48.216Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ee8f8470-830e-48ff-a727-0c31c27777bc", "object": "cross-site-replica-proxysql", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:53:48.340Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ee8f8470-830e-48ff-a727-0c31c27777bc", "object": "cross-site-replica-proxysql-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T02:53:49.318Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "d2b464ed-300c-4c75-b2eb-16737cb6c44f", "object": "cross-site-replica-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T02:53:49.348Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "d2b464ed-300c-4c75-b2eb-16737cb6c44f", "object": "cross-site-replica-proxysql", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T02:53:52.777Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "b9cec4f2-c99b-4df5-95d0-0735ddc2f939"} 2026-07-02T02:53:57.949Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "82fade73-09a9-471c-be90-7bc23970248e"} 2026-07-02T02:54:03.571Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "58b4476c-7e67-42c5-9059-0293d6359fc2"} 2026-07-02T02:54:08.835Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "733b2625-b9ce-4316-a625-ec963d33f0f5"} 2026-07-02T02:54:14.332Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "a895a575-6d94-40fd-a21c-8f7e930e1d17"} 2026-07-02T02:54:19.429Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "59933def-05b2-46ec-8a22-6886dee5b0ac"} 2026-07-02T02:54:25.178Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "668423a2-6e6a-4dfb-8214-c0e2fac74dde"} 2026-07-02T02:54:29.848Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e0114cdf-464c-43b1-bf1e-2579049838a1"} 2026-07-02T02:54:35.475Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3633160d-0781-4360-950d-cd94c07a25ca"} 2026-07-02T02:54:40.750Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "786ae5ec-6ee5-4a12-920b-2de27a4f920d"} 2026-07-02T02:54:46.135Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1e51af83-b7bf-4373-97ce-f227348c2b41"} 2026-07-02T02:54:51.444Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d5544460-4c57-4dc3-b929-dad464de68d6"} 2026-07-02T02:54:56.556Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "2831a402-dac0-45fb-8cf4-424aff6f9de0"} 2026-07-02T02:55:01.734Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "317e8d78-8c55-406c-b721-4344744a2628"} 2026-07-02T02:55:07.359Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "57158777-0c4d-4579-a2d7-a659f01418e3"} 2026-07-02T02:55:09.218Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c47df68b-7597-4ba7-864b-6bfbc25c9f39", "user": "operator"} 2026-07-02T02:55:09.248Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c47df68b-7597-4ba7-864b-6bfbc25c9f39", "user": "monitor"} 2026-07-02T02:55:09.295Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c47df68b-7597-4ba7-864b-6bfbc25c9f39"} 2026-07-02T02:55:09.324Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c47df68b-7597-4ba7-864b-6bfbc25c9f39"} 2026-07-02T02:55:09.359Z INFO monitor user audit admin privilege granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c47df68b-7597-4ba7-864b-6bfbc25c9f39"} 2026-07-02T02:55:09.400Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c47df68b-7597-4ba7-864b-6bfbc25c9f39", "user": "xtrabackup"} 2026-07-02T02:55:09.453Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c47df68b-7597-4ba7-864b-6bfbc25c9f39"} 2026-07-02T02:55:09.483Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c47df68b-7597-4ba7-864b-6bfbc25c9f39", "user": "replication"} 2026-07-02T02:55:09.495Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c47df68b-7597-4ba7-864b-6bfbc25c9f39", "err": "get primary pxc pod: not found"} 2026-07-02T02:55:12.541Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "92c8bb2e-d6c7-4176-97d4-1223b973c691"} 2026-07-02T02:55:14.661Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "939cbf13-22b3-411e-ba2c-b16744e2a2a3", "err": "get primary pxc pod: not found"} 2026-07-02T02:55:17.849Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "edac47d0-ead6-4b1c-a41d-8a09f3866932"} 2026-07-02T02:55:20.163Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "2a1df260-d035-495a-993a-086f8025bbf5", "err": "get primary pxc pod: not found"} 2026-07-02T02:55:23.245Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "fbd718df-6aba-465e-a401-4db2ca8739d9"} 2026-07-02T02:55:28.046Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "932a00bf-a444-431c-889e-b1201931722e"} 2026-07-02T02:55:33.551Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "a7ace0a5-823c-4ca3-96da-c670a9db95f5"} 2026-07-02T02:55:38.955Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "6c5b829c-ca5b-4b0c-9cbc-c21d65406cc8"} 2026-07-02T02:55:44.153Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "38afd609-a49d-463d-89c5-3d4aa40aa8d4"} 2026-07-02T02:55:49.325Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ac17ed3a-34e0-46b0-baa8-170074e1f7d2"} 2026-07-02T02:55:55.147Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "dddd76f9-0ab0-45e4-8d7e-696f2de54251"} 2026-07-02T02:56:00.247Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "0adddf7c-921e-4efe-a7d2-8f7118cafedb"} 2026-07-02T02:56:05.379Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "27371669-76ef-4299-a000-172ee7162928"} 2026-07-02T02:56:10.675Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "4b1c8489-c7a2-4c1a-9f50-1e7b005ff71d"} 2026-07-02T02:56:16.051Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "8b5c90be-3450-483a-9f01-4012119e2a0e"} 2026-07-02T02:56:21.382Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7d7c11d2-0d47-4958-89b6-cb599d1a16f6"} 2026-07-02T02:56:26.650Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "0c5928ca-864b-4e16-bdc5-fae88838784e"} 2026-07-02T02:56:31.840Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "c0682d5b-674d-4363-bf25-5a30fe5300dc"} 2026-07-02T02:56:37.724Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7298b9e6-ee83-493a-b47d-d713bba6252f"} 2026-07-02T02:56:42.375Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "5b303b51-42b9-4f05-a77c-70cbb74c1726"} 2026-07-02T02:56:48.022Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7d5f44c0-e4d1-47bb-8b95-5753a4ea2958"} 2026-07-02T02:56:53.063Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "78e10783-bfe8-43c6-b00b-f394abd7a9cf"} 2026-07-02T02:56:58.348Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "b2c08e85-391a-4aeb-949b-8210d180751f"} 2026-07-02T02:57:03.574Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "f9164885-a3ab-440b-ab06-000de1d35d34"} 2026-07-02T02:57:09.323Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "da7456d2-7165-45b6-8dde-ab92510c1375"} 2026-07-02T02:57:14.078Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "b5b7719b-df87-45bd-873d-44d319c10439"} 2026-07-02T02:57:19.377Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "15f3c93f-42a6-4adc-9033-ca871054c070"} 2026-07-02T02:57:24.666Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "066503a8-d04f-4682-a5cd-4cf19aad2de0"} 2026-07-02T02:57:29.981Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3faaff53-2893-45df-b9cf-32735b60801d"} 2026-07-02T02:57:35.202Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "20d7c1ee-5796-4853-869e-1bb2e428bd17"} 2026-07-02T02:57:40.820Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "6aa78caa-e122-464c-b8a2-4637c702c055"} 2026-07-02T02:57:43.952Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "e4346028-1b19-4ba6-a41c-7611f192063a", "user": "root"} 2026-07-02T02:57:44.107Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "e4346028-1b19-4ba6-a41c-7611f192063a", "new version": "8.4.7-7.1"} 2026-07-02T02:57:46.052Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "4b1ce280-ed07-4c34-ac19-4e452f58a74d"} 2026-07-02T02:57:51.261Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "aea6d429-a3a1-49fb-aa3e-c650f9779868"} 2026-07-02T02:57:56.630Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "536a65b0-c72f-4885-b97a-3aeb05165e76"} 2026-07-02T02:58:01.874Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "b7f597b3-087f-405b-b82d-7685427ef472"} 2026-07-02T02:58:07.650Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "f1aa9b71-7c5d-4d8c-82ec-db52b6993788"} 2026-07-02T02:58:12.081Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "206a62e4-77f2-4591-a2d6-03bb048e6978"} 2026-07-02T02:58:15.961Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "eb73eaaf-25d6-46f7-a678-e6bbeb3785df"} 2026-07-02T02:58:21.267Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c6754391-0c0a-481e-884e-0627c9a0b872"} 2026-07-02T02:58:26.438Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "25165001-b87c-4e1f-87b5-712cb24c434d"} 2026-07-02T02:58:31.542Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "a00537d0-c3b0-46ef-9436-d142eb0b3a26"} 2026-07-02T02:58:37.473Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "fdb2ee2c-dffb-4922-9bb7-dabda46ec9a6"} 2026-07-02T02:58:42.146Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "847c431f-dc8a-4eca-8586-157a1b3f1a50"} 2026-07-02T02:58:47.241Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "bd1597d6-8895-498a-bf0f-1b2aa1053651"} 2026-07-02T02:58:52.653Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "aa455540-bc0f-4e2a-a985-d3e25d4b2498"} 2026-07-02T02:58:58.029Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c6c6330c-47b2-4e6c-b9e7-7cff39fd9ef9"} 2026-07-02T02:59:03.533Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "bf99d7d2-3696-45f1-bbcb-764d86263ca7"} 2026-07-02T02:59:09.355Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "5ad4da40-dde8-41a9-8340-1876d0b9e1fb"} 2026-07-02T02:59:14.439Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "3e2919a6-d9d9-487a-b1ef-7c32e3d25ab9"} 2026-07-02T02:59:19.747Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "0f9531e4-8a16-4c22-a27f-c984ebb89511"} 2026-07-02T02:59:24.952Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "d7a6407d-f49f-48b0-ada3-af41990c01bf"} 2026-07-02T02:59:29.948Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "7204f1d7-600b-4dc5-ac7d-b2a1039440e8"} 2026-07-02T02:59:35.633Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "9be94b3f-17af-4fd8-bc65-5d6f8880d9cd"} 2026-07-02T02:59:41.634Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "1c002a66-97db-481b-a89b-29eb6d3aec23"} 2026-07-02T02:59:46.779Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "714a11ff-229b-4c3f-b835-e5ad65bd4925"} 2026-07-02T02:59:52.143Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "d6cee022-67f7-4a00-90a8-63c955b550bf"} 2026-07-02T02:59:57.270Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "9343ef35-116f-4347-a3d6-85b00d979835"} 2026-07-02T03:00:02.565Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "fa7de1fb-efe6-4d6c-be6a-7b6e5411e468"} 2026-07-02T03:00:07.872Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "1e163d10-9092-467f-96e7-0860dcee7a80"} 2026-07-02T03:00:12.574Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "dd17656a-58aa-4195-8f99-a6b95b8ae91a"} 2026-07-02T03:00:18.272Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "d4bff3d0-70cf-47c7-9493-09953e5f3418"} 2026-07-02T03:00:24.036Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "691b4555-08cd-48f0-a511-cb818b554e8f"} 2026-07-02T03:00:28.948Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "2da1929a-5776-406b-bf2c-787344e85c23"} 2026-07-02T03:00:34.553Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "3b72bbd3-c68b-40af-b008-28fe92197ace"} 2026-07-02T03:00:39.638Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "cf27fc50-ff52-4fcb-985c-97923fa92e23"} 2026-07-02T03:00:44.784Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "5f23c82b-c8c4-4050-8b6f-9211efe40b8a"} 2026-07-02T03:00:50.350Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "0a0e3b46-80b2-4cf7-bc92-19353287d638"} 2026-07-02T03:00:55.635Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8f36b9e7-7c5c-413d-88b5-349bb4b8672b"} 2026-07-02T03:01:00.836Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "2de8b0cf-94fa-4eac-835d-3ef8d5c23373"} 2026-07-02T03:01:04.078Z INFO stopping cluster {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "503caaa8-40a7-443e-83b8-944cf6ca1ff8", "cluster": "cross-site-replica"} 2026-07-02T03:01:04.186Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8d92111d-6b3e-4796-b00a-ddc94513a10c", "object": "cross-site-replica-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:01:04.244Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8d92111d-6b3e-4796-b00a-ddc94513a10c", "object": "cross-site-replica-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:01:04.415Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8d92111d-6b3e-4796-b00a-ddc94513a10c", "object": "cross-site-replica-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:01:06.770Z ERROR sync users {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "2bb8c2e4-c244-4b9d-837f-bcf0337d8cf8", "error": "exec syncusers: failed to execute command in pod: unable to upgrade connection: pods \"cross-site-replica-proxysql-2\" not found / / ", "errorVerbose": "exec syncusers: failed to execute command in pod: unable to upgrade connection: pods \"cross-site-replica-proxysql-2\" not found / / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:1000\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:859\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T03:01:12.450Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "c06b27d4-aed5-4db0-9427-eb97b18f6ec6"} 2026-07-02T03:01:17.952Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "a398c8e5-5cb5-4863-8aa5-91374444f0f9"} 2026-07-02T03:01:23.030Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "15cc76ef-a6cc-4ca7-9db8-c1c46bc231c9"} 2026-07-02T03:01:28.783Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "03c473cb-4e0a-48b6-8b5c-ba2e423f48e3"} 2026-07-02T03:01:33.954Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "fb5fb6c4-aa4c-4d77-99b1-c60cfccf32e8"} 2026-07-02T03:01:39.276Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "981197b6-32e2-4005-a1a7-6aadc3528140"} 2026-07-02T03:01:44.656Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ad1d81d3-8d7e-40c4-a45c-0b21a55725cc"} 2026-07-02T03:01:46.260Z INFO starting restore {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "33d3d173-55ba-48d3-8bb6-1cc72c88674d", "cluster": "cross-site-replica", "backup": ""} 2026-07-02T03:01:46.333Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "e88c59e3-a6d6-47b2-82bc-cf7c6c2dbaeb", "job": "restore-job-backup-minio-cross-site-replica"} 2026-07-02T03:01:50.329Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "5ce26c97-d3a4-41bb-a91f-d7b1be545a0b"} 2026-07-02T03:01:51.352Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "0382db85-b547-41f2-a3f1-3fc02cb4f20d", "job": "restore-job-backup-minio-cross-site-replica"} 2026-07-02T03:01:55.168Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7ec94bfe-0a73-473d-8a58-4c839a77a9e5"} 2026-07-02T03:01:56.369Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "76a45c14-fe09-47ab-925d-d899743f455e", "job": "restore-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:01.063Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "569619dc-b3ea-4be4-99a7-17fcd973cfff"} 2026-07-02T03:02:01.386Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "7bb58bf3-4bb4-45c8-b49d-cafc17d9b164", "job": "restore-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:06.352Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d1cf4dd0-a5c8-49ca-b344-a99c07e0dc0b"} 2026-07-02T03:02:06.407Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "685ee196-888b-461a-9869-632f5a391ef9", "job": "restore-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:11.427Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "392a12e8-31b9-4461-85c5-dcc092808155", "job": "restore-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:11.453Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "91579aa3-f505-4f13-8661-6cda356618a0"} 2026-07-02T03:02:16.444Z INFO preparing cluster {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "33f76272-7084-4f73-b0f2-8a5088d3b33d", "cluster": "cross-site-replica"} 2026-07-02T03:02:16.513Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "bedb277e-f44d-4904-ab3e-04c40ba92f0d", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:16.676Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "a020f62f-d294-4cf6-a58c-3699864772ed"} 2026-07-02T03:02:21.534Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "955d0300-c2e0-435c-8c56-3b23ed1e46a3", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:22.540Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "05ef3486-8b48-429c-942d-288fec622cdd"} 2026-07-02T03:02:26.551Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "3ffba61a-73ed-477b-9d06-0feba52d9a60", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:27.277Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e9d0a03d-c2f7-4fd3-95c1-41558b12676c"} 2026-07-02T03:02:31.565Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "11949352-fbf0-4b18-9dd3-d28781a41690", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:32.251Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "55bd95e5-f76a-4ee2-ae33-6eaa1281dee3"} 2026-07-02T03:02:36.582Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "9d94ee04-39d1-41a3-81b0-838075f677ac", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:37.775Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ff1dbc7f-cda9-4787-b32e-5529b00cd418"} 2026-07-02T03:02:41.597Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "c28eb785-e423-4d9b-a894-84c65aeab28f", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:43.035Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e0f538dd-e3ca-4b5c-8255-ea90d3f7c8b7"} 2026-07-02T03:02:46.627Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "924ecedb-65ea-46ed-b493-618387d4e634", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:48.260Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ee2d011f-1c86-437e-86de-b190702354e8"} 2026-07-02T03:02:51.644Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "28fd0ba9-f60f-4719-bbb8-0a94c7c0652b", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:53.655Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7f4000d1-a988-4775-939b-924208ea6acf"} 2026-07-02T03:02:56.660Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "ad3260a1-810c-45df-a8d7-7433fb8d4822", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:02:59.158Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "72a6b940-fe21-40f4-b67a-451c03dfcb9a"} 2026-07-02T03:03:01.711Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "65240402-b998-4438-91d6-a4ba3b728bbc", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:03:04.559Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7bacf571-5038-48e1-844b-2f22902a6487"} 2026-07-02T03:03:06.730Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "efff0aee-8bd4-4f90-9f5c-56e704b2ccc4", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:03:09.564Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3e27ecc0-abe5-44ba-90df-7bec29306b53"} 2026-07-02T03:03:11.753Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "0e7b92d3-34a4-4c62-997c-f2b62a698ff8", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:03:15.078Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "f0083bd0-1d3e-4a08-ad7f-d2818d98599f"} 2026-07-02T03:03:16.775Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "95bd1bc0-19b1-4fa2-981f-c00aa3febd17", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:03:19.937Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "dd427ef8-44e7-40ea-b3a1-1da2eb5ee8af"} 2026-07-02T03:03:21.793Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "4a8cb398-e4d3-4fc1-bfa8-fbcdddc8825a", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:03:25.749Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "0fc6c255-6b0e-4577-9879-7792e158c022"} 2026-07-02T03:03:26.809Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "363b439d-66f7-4458-9dd3-46d323359509", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:03:30.968Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1ec40b4f-1693-4847-8385-30d6d2a47056"} 2026-07-02T03:03:31.832Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "f9dea5c9-d765-4f33-a269-44fd6d6f126f", "job": "prepare-job-backup-minio-cross-site-replica"} 2026-07-02T03:03:36.322Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "07ac1d6d-efdf-40a4-b07e-899798f2d947"} 2026-07-02T03:03:36.847Z INFO starting cluster {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "603d4d5f-0666-410a-a223-4f0c11ffde76", "cluster": "cross-site-replica"} 2026-07-02T03:03:37.051Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "e6502d0c-a6c2-40d7-a9c5-911b1052decc", "object": "cross-site-replica-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:03:37.105Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "e6502d0c-a6c2-40d7-a9c5-911b1052decc", "object": "cross-site-replica-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:03:37.222Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "e6502d0c-a6c2-40d7-a9c5-911b1052decc", "object": "cross-site-replica-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:03:41.826Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "441927d7-35bd-4b27-bf3e-5e57804f4995"} 2026-07-02T03:03:41.960Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "6f87fb90-2cb5-48cc-b8b9-4fac1037f661", "cluster": "cross-site-replica"} 2026-07-02T03:03:46.978Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "5c5f0898-5fd2-478c-86bc-36921178d8fa", "cluster": "cross-site-replica"} 2026-07-02T03:03:47.637Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "11a2b78b-9879-4610-b36a-3e1ee4b7d618"} 2026-07-02T03:03:52.006Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "bc96cc4a-1fd9-499d-8f4f-60e9614c12e7", "cluster": "cross-site-replica"} 2026-07-02T03:03:52.768Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "bb16dec2-9bbf-4565-bc9d-e8d9f392fa86"} 2026-07-02T03:03:57.060Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "4501a810-fcb1-4098-826f-4db406495e1a", "cluster": "cross-site-replica"} 2026-07-02T03:03:57.647Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "eba00b8e-209c-44fa-aed1-8cc588fa4c94"} 2026-07-02T03:04:02.077Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "ac636d84-935a-4fe0-b30c-142a9293e07b", "cluster": "cross-site-replica"} 2026-07-02T03:04:03.253Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "f15ac470-22b3-4efd-b30a-45f76a10348f"} 2026-07-02T03:04:07.167Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "21a28aa3-b359-4244-a36e-f31ce8c594fc", "cluster": "cross-site-replica"} 2026-07-02T03:04:08.857Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "f5100452-acfa-4e3a-acf2-00775046401c"} 2026-07-02T03:04:12.186Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "19a0d6d0-8b22-4a92-8ba3-43fd57edb75b", "cluster": "cross-site-replica"} 2026-07-02T03:04:14.123Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "a0b2d571-3455-48a7-bd20-f1eafed13bc0"} 2026-07-02T03:04:17.205Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "ef06fd60-cdc4-43a1-a10e-0b179383a857", "cluster": "cross-site-replica"} 2026-07-02T03:04:19.456Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "eebf31dc-8de8-4eea-9aca-03d1ae7fb711"} 2026-07-02T03:04:22.242Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "f8b05b70-c7a0-42ea-aaa6-aabab10f9163", "cluster": "cross-site-replica"} 2026-07-02T03:04:24.577Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "0f62791c-4b86-4f16-8871-0892f064a78d"} 2026-07-02T03:04:27.264Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "7fe276f8-842d-4011-b4db-6e76d6cf64bb", "cluster": "cross-site-replica"} 2026-07-02T03:04:29.948Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "8920ad1a-744a-4a29-a65e-6da9b0cdf7ae"} 2026-07-02T03:04:32.295Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "705734bd-3174-4200-aeda-71e86c9a113a", "cluster": "cross-site-replica"} 2026-07-02T03:04:35.358Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "028f9f3e-fbf3-41f8-86b2-20fa5ed05342"} 2026-07-02T03:04:37.341Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "0455e222-7399-4b41-a5da-d455cdb2c821", "cluster": "cross-site-replica"} 2026-07-02T03:04:40.634Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "c8fc2b64-fd28-42ce-a1aa-35d0a9447804"} 2026-07-02T03:04:42.355Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "cbd5d6be-3230-4cb4-b743-08954db1f81f", "cluster": "cross-site-replica"} 2026-07-02T03:04:46.046Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "66af5020-3401-43d2-ba3b-66bdb39b6013"} 2026-07-02T03:04:47.374Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "f166e5cb-5232-4755-97e8-06b7e30ad742", "cluster": "cross-site-replica"} 2026-07-02T03:04:51.079Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "9dd63d13-4c29-4609-9d6f-6bf570e9cca7"} 2026-07-02T03:04:52.388Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "1185d495-f3a5-4ec0-8265-b4e0b0b0b08b", "cluster": "cross-site-replica"} 2026-07-02T03:04:56.450Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ee699bc2-6dfa-4adf-a4e5-a97780246e84"} 2026-07-02T03:04:57.403Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "985f6a80-b685-436b-8d90-0bd830d66b82", "cluster": "cross-site-replica"} 2026-07-02T03:05:01.453Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "4dcfa905-3711-412c-b96a-ccf8ffda2b8b"} 2026-07-02T03:05:02.420Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "0477605a-d9f5-492c-9b1c-1695b4b969e7", "cluster": "cross-site-replica"} 2026-07-02T03:05:07.221Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e840bfa9-7b0f-4410-ba7c-af3f345d52c5"} 2026-07-02T03:05:07.439Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "7d42cab6-c98a-4a75-923c-fdcaccbf34e1", "cluster": "cross-site-replica"} 2026-07-02T03:05:11.756Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "f51875a1-ed92-481e-9b5a-ee6ffd65bb1d"} 2026-07-02T03:05:12.460Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "92de6da1-f378-478c-8744-fc79e130b135", "cluster": "cross-site-replica"} 2026-07-02T03:05:17.448Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "4904ed14-d984-4402-848c-81a5ac9f6248"} 2026-07-02T03:05:17.484Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "37ca176e-1cc8-49bc-9400-77db469526d0", "cluster": "cross-site-replica"} 2026-07-02T03:05:22.501Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "efce02ef-989b-499f-aad8-204affdd476b", "cluster": "cross-site-replica"} 2026-07-02T03:05:22.952Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "248a735b-76e9-4d3a-8c69-a95915d84670"} 2026-07-02T03:05:27.531Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "00840b29-c0a8-45e1-bc60-f321109e64bb", "cluster": "cross-site-replica"} 2026-07-02T03:05:27.852Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "35b0f11b-4b33-4527-b1d7-c1238615ea05"} 2026-07-02T03:05:32.551Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "80bfcf9b-73a7-4344-b50f-124bb0235a79", "cluster": "cross-site-replica"} 2026-07-02T03:05:33.175Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "9761d62c-ea61-48a6-9a16-c91c0873ec26"} 2026-07-02T03:05:37.567Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "9c0bef9d-5faa-4e82-b87d-58d8648922bf", "cluster": "cross-site-replica"} 2026-07-02T03:05:38.925Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "512ac9d8-66c9-4bb7-91bb-b733c1de1990"} 2026-07-02T03:05:42.583Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "ba92a929-0878-4082-92ec-2a1573f9bfb0", "cluster": "cross-site-replica"} 2026-07-02T03:05:44.344Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "5e33a55f-b4b0-46bb-af41-2f36ff37397e"} 2026-07-02T03:05:47.601Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "6a8716db-00c1-4f09-84ed-d3edb66fabe6", "cluster": "cross-site-replica"} 2026-07-02T03:05:49.575Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3828809a-a74f-4016-8193-3f75ccb4a86a"} 2026-07-02T03:05:52.616Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "7076de5f-22f6-44e7-85d8-412ff4be256a", "cluster": "cross-site-replica"} 2026-07-02T03:05:54.745Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "4c49d3d9-10ae-41b9-a234-c64edc5f860c"} 2026-07-02T03:05:57.636Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "908cce58-820b-4821-be64-54b7cf743e7c", "cluster": "cross-site-replica"} 2026-07-02T03:05:59.669Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "11e8e7c5-5649-4934-b185-464ad7a8b6a1"} 2026-07-02T03:06:02.657Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "1526e818-4d4d-4b3c-8c3b-956421987261", "cluster": "cross-site-replica"} 2026-07-02T03:06:05.345Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "597796dd-91d9-4cb7-a38e-6802e8918a58"} 2026-07-02T03:06:07.674Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "9650852e-21f8-4e50-a165-86167f04211d", "cluster": "cross-site-replica"} 2026-07-02T03:06:10.548Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1276f1bf-73b3-456c-a3e8-ed697cdb23d1"} 2026-07-02T03:06:12.708Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "7767dd1c-4cd8-484c-8485-a9c5c99f9fd7", "cluster": "cross-site-replica"} 2026-07-02T03:06:15.526Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "0b3a8bf3-d86c-452f-a1a6-c0d886c9946f"} 2026-07-02T03:06:17.725Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "5af92076-ee22-4499-90c0-7a108f8e0946", "cluster": "cross-site-replica"} 2026-07-02T03:06:21.350Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "5064b5cf-28ad-4749-8061-7bad346c8734"} 2026-07-02T03:06:22.745Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "d8c9810b-56d5-4562-a199-c9fca257bf0f", "cluster": "cross-site-replica"} 2026-07-02T03:06:26.435Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "f4fabe65-d275-4c75-856f-400d68602622"} 2026-07-02T03:06:27.761Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "e39213d1-5830-4d2a-82c5-19b2f0cfc9fd", "cluster": "cross-site-replica"} 2026-07-02T03:06:31.762Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ad0f28f2-de7a-4b40-a0a5-1b74689f0a12"} 2026-07-02T03:06:32.777Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "475d6c44-e034-4da2-b61d-c976382b5c0d", "cluster": "cross-site-replica"} 2026-07-02T03:06:37.545Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1a94b324-c7dc-4e4d-9080-2915f001111c"} 2026-07-02T03:06:37.797Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "44b920bf-8e1c-4c83-b288-b0ea7ac9866c", "cluster": "cross-site-replica"} 2026-07-02T03:06:42.036Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "b2b8668f-d68c-4afc-8bd1-1d99947d130c"} 2026-07-02T03:06:42.816Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "176d2f47-d7d4-4797-9439-700987b9c9d9", "cluster": "cross-site-replica"} 2026-07-02T03:06:47.380Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "b8e6686f-eef5-4429-972a-39110f1cd408"} 2026-07-02T03:06:47.832Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "df947ecc-6c50-40ad-8e88-17f1859cad78", "cluster": "cross-site-replica"} 2026-07-02T03:06:52.844Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "54cc5ef1-50a1-45df-be86-25021d1c5d27"} 2026-07-02T03:06:52.854Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "02ef6176-d78e-47be-9373-cc141e0d75d9", "cluster": "cross-site-replica"} 2026-07-02T03:06:57.873Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio", "reconcileID": "e35ef8af-eba7-4b64-892c-e607577861e3", "cluster": "cross-site-replica"} 2026-07-02T03:06:58.152Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "a99d8a56-6660-45d4-ba54-6ba3d95ff953"} 2026-07-02T03:07:03.477Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ca614004-d37e-4a5c-b9d3-96cb47cf7089"} 2026-07-02T03:07:08.521Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7370de37-c784-47b1-8caa-9752adf2ef3a"} 2026-07-02T03:07:14.146Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "f8473c9f-b201-43de-b371-2f340c2cc7ef"} 2026-07-02T03:07:19.458Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "2f5cf1ce-066c-4c23-a292-2f12789807b4"} 2026-07-02T03:07:24.584Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "4cd5e5e8-46c8-4cd5-9475-2265a34594de"} 2026-07-02T03:07:29.961Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1358590e-5f5f-4cf6-b955-9d6e9649bbe5"} 2026-07-02T03:07:35.755Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "2bf49baa-e8b2-47c9-8922-ff04288e61f4"} 2026-07-02T03:07:40.326Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "dd625005-756d-4e03-9c07-ce27c5e8aa26"} 2026-07-02T03:07:45.522Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e54e9f7d-0464-4c96-b4a5-a618ee842912"} 2026-07-02T03:07:50.642Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "b4b5815a-64d8-46cc-92ca-2611b8b93c5f"} 2026-07-02T03:07:56.078Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d7478fa4-4940-4f88-adef-a0265e2a06cf"} 2026-07-02T03:08:01.075Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "45f12771-4755-408e-8f04-bdf7abc55226"} 2026-07-02T03:08:07.625Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "607a3909-7001-4160-9ce4-1d45064a0f2d"} 2026-07-02T03:08:07.827Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "691ba6a3-233d-4fd1-82e1-cdb9a7f3e124", "object": "cross-site-replica-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:08:07.889Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "691ba6a3-233d-4fd1-82e1-cdb9a7f3e124", "object": "cross-site-replica-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:08:08.067Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "691ba6a3-233d-4fd1-82e1-cdb9a7f3e124", "pod": "cross-site-replica-pxc-2"} 2026-07-02T03:08:08.075Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "691ba6a3-233d-4fd1-82e1-cdb9a7f3e124", "pod": "cross-site-replica-pxc-0"} 2026-07-02T03:08:08.083Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "691ba6a3-233d-4fd1-82e1-cdb9a7f3e124", "pod": "cross-site-replica-pxc-1"} 2026-07-02T03:08:08.150Z INFO Replication pod has changed {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "691ba6a3-233d-4fd1-82e1-cdb9a7f3e124", "new replication pod": "cross-site-replica-pxc-0"} 2026-07-02T03:08:10.803Z ERROR sync users {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "691ba6a3-233d-4fd1-82e1-cdb9a7f3e124", "error": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") / / ", "errorVerbose": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") / / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:1000\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:859\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T03:08:14.078Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "72ccbb40-af9e-4967-bea6-c64fb53be87f", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-proxy' (using password: YES)"} 2026-07-02T03:08:15.530Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "bbd36c4a-b606-40b1-acbc-9dc2081f8c48", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-proxy' (using password: YES)"} 2026-07-02T03:08:17.772Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "20ae6fb6-08d4-48dc-b11f-63d4dc94686d"} 2026-07-02T03:08:23.420Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "0bedaa6d-98aa-47b5-a661-8f554c7ff66e"} 2026-07-02T03:08:26.828Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "67b39bd8-485b-4365-a60d-0802bd63f3ad", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-proxy' (using password: YES)"} 2026-07-02T03:08:30.839Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "28c1c86f-5e13-4469-bdd0-5250574976be"} 2026-07-02T03:08:32.160Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "435f9755-e2b8-4613-b00f-bce34b880ec7", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-proxy' (using password: YES)"} 2026-07-02T03:08:36.835Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7816280e-ad9b-4fd6-b650-32fe9d4f7d20"} 2026-07-02T03:08:37.563Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "0fd5f61f-cf25-413a-8e8f-1fcd777116e2", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-proxy' (using password: YES)"} 2026-07-02T03:08:38.385Z INFO Password changed, updating user {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "43a360af-1c57-4b0e-8c2b-500d117fb91e", "user": "replication"} 2026-07-02T03:08:38.400Z INFO Password updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "43a360af-1c57-4b0e-8c2b-500d117fb91e", "user": "replication"} 2026-07-02T03:08:38.427Z INFO MySQL init secret created {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "43a360af-1c57-4b0e-8c2b-500d117fb91e", "secret": "cross-site-replica-mysql-init", "user": "replication"} 2026-07-02T03:08:38.454Z INFO Internal secrets updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "43a360af-1c57-4b0e-8c2b-500d117fb91e", "user": "replication"} 2026-07-02T03:08:38.466Z INFO Old password discarded {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "43a360af-1c57-4b0e-8c2b-500d117fb91e", "user": "replication"} 2026-07-02T03:08:38.496Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "43a360af-1c57-4b0e-8c2b-500d117fb91e", "channel": "source_to_replica", "Last_IO_Error": ""} 2026-07-02T03:08:40.009Z ERROR sync users {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "43a360af-1c57-4b0e-8c2b-500d117fb91e", "error": "syncusers: WARNING (line:1652) : Ignoring query rule for replication, since this is not using singlewrite mode (max-writers is not 1)\n", "errorVerbose": "syncusers: WARNING (line:1652) : Ignoring query rule for replication, since this is not using singlewrite mode (max-writers is not 1)\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:1004\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:859\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T03:08:44.161Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "ff930291-060d-44b4-bd2a-149af22274ea"} 2026-07-02T03:08:48.426Z ERROR sync users {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "3d9b1ae1-3026-4652-8819-d36123fcd02d", "error": "syncusers: WARNING (line:1652) : Ignoring query rule for replication, since this is not using singlewrite mode (max-writers is not 1)\n", "errorVerbose": "syncusers: WARNING (line:1652) : Ignoring query rule for replication, since this is not using singlewrite mode (max-writers is not 1)\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:1004\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:859\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T03:08:53.466Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c2342964-10bb-4084-b2b4-0614198d611a"} 2026-07-02T03:08:59.336Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "74d612e9-48e3-478e-8456-6a3339db9307"} 2026-07-02T03:09:05.072Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "35a516c1-ea72-4ff2-84a9-4ce23d48018b"} 2026-07-02T03:09:10.448Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c13e383e-f117-436c-8596-4cdbeb7de6bb"} 2026-07-02T03:09:15.758Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b977eb00-997a-4c05-b98a-bc01d0c81d2f"} 2026-07-02T03:09:21.350Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "bcf59ef5-3085-4390-8c96-cd01a47165cd"} 2026-07-02T03:09:26.148Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "15c917d5-9d07-43c5-92e5-17f8767a3a5a"} 2026-07-02T03:09:31.780Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "010ac13c-cf11-4433-84ed-a04eeb2e8a96"} 2026-07-02T03:09:37.843Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "91e454f3-a66a-4c9d-bc1c-7dc1d6ddb05e"} 2026-07-02T03:09:42.473Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "369c10d4-af55-402c-b175-ab2c7c70744a"} 2026-07-02T03:09:48.241Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "06109f0e-9c27-4939-becf-66e8ac285888"} 2026-07-02T03:09:53.551Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b03325fa-f00d-4f6d-a56d-055447e23c8c"} 2026-07-02T03:09:58.951Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "7ed2d267-5746-4257-9729-d9ef0369b0d6"} 2026-07-02T03:10:04.133Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "2b6fd753-4f0c-497f-badf-bf1bd5613515"} 2026-07-02T03:10:10.031Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "97f492dc-eebe-4149-8113-b3d8022310e8"} 2026-07-02T03:10:15.080Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "7abbfed0-6a0b-4aa4-873c-8300798c9bc1"} 2026-07-02T03:10:21.260Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b710dc54-fc7d-4901-aba9-9189b7b74907"} 2026-07-02T03:10:26.076Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "f5da794f-eb56-45d1-9ac8-51522328fcc3"} 2026-07-02T03:10:31.470Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "2ae256dd-c3fd-4cf3-801d-015534d7e82d"} 2026-07-02T03:10:37.079Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "da5da111-d2cc-444e-ad91-c6c623c6f35f"} 2026-07-02T03:10:42.331Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "e2753562-880c-423b-814a-de2013047c1e"} 2026-07-02T03:10:47.730Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "f33e93a3-d89f-4a9a-a22e-fd02a9ada04e"} 2026-07-02T03:10:53.065Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "593ae46b-7bf4-440c-992f-0d1368eb4c81"} 2026-07-02T03:10:58.055Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "f4dbfbb6-c232-4350-8055-5b236eb7d563"} 2026-07-02T03:11:04.071Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b8867bb5-d59a-4f53-8c00-9cf820272757"} 2026-07-02T03:11:09.140Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "36bf8512-4866-46c9-8f8b-f80dda55e998"} 2026-07-02T03:11:14.631Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ca60b571-86c0-45d4-b257-18285f3b44e0"} 2026-07-02T03:11:20.147Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "4ca9324c-0ba5-4136-89ca-f27d2ddf935e"} 2026-07-02T03:11:25.132Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "861e790a-1118-4d00-81ea-a967c7487174"} 2026-07-02T03:11:31.135Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b871abfe-f0e4-4a02-ae5a-0072c4830b12"} 2026-07-02T03:11:36.363Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "5fb7e0e7-9637-46e5-9782-6108e8fe5d4d"} 2026-07-02T03:11:41.377Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "280d1b9d-2e00-4e06-9170-b6e7e72739cd"} 2026-07-02T03:11:46.763Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "9da5700a-a098-4098-9a56-c6c7cd5d5381"} 2026-07-02T03:11:52.259Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c80fe4e2-2ccf-4389-8e09-ea225d942ee8"} 2026-07-02T03:11:53.321Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio-replica", "reconcileID": "a17033d7-4384-432d-bbed-76293bc70278", "cluster": "cross-site-replica", "storage": "minio", "allowed": true} 2026-07-02T03:11:53.353Z INFO Created a new backup job {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio-replica", "reconcileID": "a17033d7-4384-432d-bbed-76293bc70278", "namespace": "cross-site-proxysql-replica-17697", "name": "xb-backup-minio-replica"} 2026-07-02T03:11:57.740Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "edf0a234-386c-4041-b4ed-a218311a5ebc"} 2026-07-02T03:11:58.354Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio-replica", "reconcileID": "48fee63d-ad26-4065-b2c4-e9058421c321", "cluster": "cross-site-replica", "storage": "minio", "allowed": true} 2026-07-02T03:11:58.509Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio-replica", "reconcileID": "9e4420b4-9912-415f-9633-31d48f0b77c6", "cluster": "cross-site-replica", "storage": "minio", "allowed": true} 2026-07-02T03:12:02.752Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "90494a67-4149-494a-885f-e9835bba44aa"} 2026-07-02T03:12:03.601Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio-replica", "reconcileID": "eff1a2af-1f59-4e5d-9d5f-2d97d8ebc4f5", "cluster": "cross-site-replica", "storage": "minio", "allowed": true} 2026-07-02T03:12:08.455Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "4903b987-f4b4-4776-a711-cfebd80a9fc2"} 2026-07-02T03:12:08.636Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio-replica", "reconcileID": "170891d2-fa29-4555-8bd9-424809d49276", "cluster": "cross-site-replica", "storage": "minio", "allowed": true} 2026-07-02T03:12:13.672Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio-replica", "reconcileID": "0d7bea9b-c0ab-40da-a186-a0c11f2a5070", "cluster": "cross-site-replica", "storage": "minio", "allowed": true} 2026-07-02T03:12:14.257Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "cad8db1a-818a-466f-a978-4273e1ba0917"} 2026-07-02T03:12:18.713Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio-replica", "reconcileID": "43533c6c-cb85-4579-86b3-8f226a744300", "cluster": "cross-site-replica", "storage": "minio", "allowed": true} 2026-07-02T03:12:19.904Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ef7b47f0-f783-47d1-871f-c6bec51544a3"} 2026-07-02T03:12:23.748Z DEBUG Check if parallel backups are allowed {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio-replica", "reconcileID": "aafe07fa-b59c-4d2a-80a5-cf9beaab1cec", "cluster": "cross-site-replica", "storage": "minio", "allowed": true} 2026-07-02T03:12:23.782Z DEBUG Removing mysql-init secret {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio-replica", "reconcileID": "aafe07fa-b59c-4d2a-80a5-cf9beaab1cec", "job": "xb-backup-minio-replica", "secret": "cross-site-replica-mysql-init"} 2026-07-02T03:12:23.782Z INFO Backup succeeded {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup-minio-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "backup-minio-replica", "reconcileID": "aafe07fa-b59c-4d2a-80a5-cf9beaab1cec", "job": "xb-backup-minio-replica"} 2026-07-02T03:12:24.848Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "5fd4eb09-ae3b-43ea-946a-fce650228200"} 2026-07-02T03:12:30.079Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b6caf11d-7b1d-406e-9cec-8255769dc91c"} 2026-07-02T03:12:31.139Z INFO stopping cluster {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "f0f97037-f5e0-41de-b2b7-a12271b01095", "cluster": "cross-site-source"} 2026-07-02T03:12:31.258Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7b850c9e-1989-4e28-93f6-d4e74e7e7126"} 2026-07-02T03:12:31.259Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7b850c9e-1989-4e28-93f6-d4e74e7e7126", "object": "cross-site-source-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:12:31.321Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7b850c9e-1989-4e28-93f6-d4e74e7e7126", "object": "cross-site-source-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:12:31.405Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7b850c9e-1989-4e28-93f6-d4e74e7e7126", "object": "cross-site-source-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:12:36.654Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "aeac0a7f-e651-45c2-9159-b09ff9f97db8"} 2026-07-02T03:12:42.366Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "88901e87-a519-4806-afda-c298aead4ea1"} 2026-07-02T03:12:47.629Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "df105aea-6dd4-4d57-b0cf-c825ccb84ad2"} 2026-07-02T03:12:52.777Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "78186dd5-f4dc-4134-a978-0c1aadbc93a1"} 2026-07-02T03:12:55.063Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ef039d9e-3c2e-4d8d-950a-d5771f5d9f13", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:12:58.358Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ef039d9e-3c2e-4d8d-950a-d5771f5d9f13"} 2026-07-02T03:13:00.643Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c68ab1aa-a5c1-4893-ab7e-8291e925da90", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:13:04.034Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c68ab1aa-a5c1-4893-ab7e-8291e925da90"} 2026-07-02T03:13:06.068Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8827c088-6243-44c2-bd94-d87425b9d9ed", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:13:11.395Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "e04fb49e-b685-474d-a950-33c3e63bbc27", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:13:14.259Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8827c088-6243-44c2-bd94-d87425b9d9ed"} 2026-07-02T03:13:16.775Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ca9beea2-bad2-4adc-8780-1af53f2d5dcc", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:13:20.451Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ca9beea2-bad2-4adc-8780-1af53f2d5dcc"} 2026-07-02T03:13:22.160Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "d2cc0ffb-d88e-48b5-b3b6-dc0ae39cc4a5", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:13:25.308Z INFO starting restore {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "6d85194b-8689-46b0-85e6-bf3ba156b6e6", "cluster": "cross-site-source", "backup": ""} 2026-07-02T03:13:25.376Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "d2cc0ffb-d88e-48b5-b3b6-dc0ae39cc4a5"} 2026-07-02T03:13:25.381Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "3835e414-0f38-47f9-b994-e7ef9d6a174a", "job": "restore-job-backup-minio-cross-site-source"} 2026-07-02T03:13:27.556Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "e5622388-403a-4093-86e1-df13d1c7ca69", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:13:30.406Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "f29e2bfd-5114-4469-a6b6-88878da4a88f", "job": "restore-job-backup-minio-cross-site-source"} 2026-07-02T03:13:30.849Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "e5622388-403a-4093-86e1-df13d1c7ca69"} 2026-07-02T03:13:32.943Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "7515e7cb-d019-4430-81ea-edbbe780e1be", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:13:35.420Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "4d0ceb3e-39d1-4075-8e5f-a79bf67af4dd", "job": "restore-job-backup-minio-cross-site-source"} 2026-07-02T03:13:36.464Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "7515e7cb-d019-4430-81ea-edbbe780e1be"} 2026-07-02T03:13:38.581Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "29324cfe-4048-469a-9ea4-4aac04e69e01", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:13:40.434Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "05aecff6-85b6-4259-9890-e4b8e6a5fc6f", "job": "restore-job-backup-minio-cross-site-source"} 2026-07-02T03:13:41.977Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "29324cfe-4048-469a-9ea4-4aac04e69e01"} 2026-07-02T03:13:43.980Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "931cc07b-aead-4a56-9814-16efc2823230", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:13:45.467Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "29802f89-89e3-4337-b45a-5751bdd793ac", "job": "restore-job-backup-minio-cross-site-source"} 2026-07-02T03:13:47.640Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "931cc07b-aead-4a56-9814-16efc2823230"} 2026-07-02T03:13:49.308Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "70c36b25-80d9-47c8-9357-127951ee0f57", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:13:50.484Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "8fa03877-c6d2-481b-9cd9-425428bc3101", "job": "restore-job-backup-minio-cross-site-source"} 2026-07-02T03:13:52.858Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "70c36b25-80d9-47c8-9357-127951ee0f57"} 2026-07-02T03:13:54.662Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "fba62b92-3c67-4529-ba95-62a4c855b2fb", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:13:55.553Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "214ebbb2-644b-434c-acc7-1f2d62815dc8", "job": "restore-job-backup-minio-cross-site-source"} 2026-07-02T03:13:58.449Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "fba62b92-3c67-4529-ba95-62a4c855b2fb"} 2026-07-02T03:14:00.055Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b2006089-b352-4b36-a2b7-56f50834ed26", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:14:00.569Z INFO Waiting for restore job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "0502ebd7-011e-446a-9fac-0fc1353bc4d1", "job": "restore-job-backup-minio-cross-site-source"} 2026-07-02T03:14:03.335Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b2006089-b352-4b36-a2b7-56f50834ed26"} 2026-07-02T03:14:05.478Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "68f28a87-6f58-4664-bafa-71d27d98770a", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:14:05.590Z INFO preparing cluster {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "76d354e1-281a-46c9-b029-943a0e0df636", "cluster": "cross-site-source"} 2026-07-02T03:14:05.670Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "76b9a77e-8009-4e07-8035-eba5272238a8", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:14:09.155Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "68f28a87-6f58-4664-bafa-71d27d98770a"} 2026-07-02T03:14:10.704Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "f9bf4dca-b877-4ec7-8453-536f42cd55e6", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:14:10.866Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "daedc05a-3a3e-412a-89d4-81a788b083d8", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:14:14.328Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "daedc05a-3a3e-412a-89d4-81a788b083d8"} 2026-07-02T03:14:15.719Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "a72dc76b-b29f-4d9d-8da8-a644221d9f96", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:14:16.190Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8c16d809-87f7-4a2d-84be-f724a06ccb17", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:14:20.006Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8c16d809-87f7-4a2d-84be-f724a06ccb17"} 2026-07-02T03:14:20.743Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "af2ba012-b6e7-4c6e-9033-0d1945c6f4bd", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:14:21.561Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8f345157-320b-451a-b00f-4bad6f1416ee", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:14:24.736Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8f345157-320b-451a-b00f-4bad6f1416ee"} 2026-07-02T03:14:25.760Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "7563224d-ca28-4f06-a357-c69e3f82cd84", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:14:26.970Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ca9e7fe3-af86-4475-a5d5-ff58d32cc421", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:14:30.083Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ca9e7fe3-af86-4475-a5d5-ff58d32cc421"} 2026-07-02T03:14:30.776Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "79cf3fad-e435-4beb-b60a-9605c1cc9bcf", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:14:32.375Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "0f03cb77-abf7-4cd3-81ef-4880ae0c7faa", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:14:35.562Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "0f03cb77-abf7-4cd3-81ef-4880ae0c7faa"} 2026-07-02T03:14:35.793Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "0851c395-f0d4-4b4f-bce0-06d4de15efff", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:14:37.758Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8cfde084-db4f-4f44-b576-05b602d5f211", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:14:40.812Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "bb7ea253-31db-4484-828d-feb810914dfd", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:14:41.230Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8cfde084-db4f-4f44-b576-05b602d5f211"} 2026-07-02T03:14:43.165Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b3059e77-ed58-4752-b5dd-7a3b4007fc09", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:14:45.827Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "3e31d03e-2ede-4769-8067-7e468715dfd6", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:14:46.358Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b3059e77-ed58-4752-b5dd-7a3b4007fc09"} 2026-07-02T03:14:48.562Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ae8cbabe-5611-4016-bb56-0c00aa4906da", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:14:50.842Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "39ec80dc-7c76-4e38-8672-b238847993bc", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:14:51.976Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ae8cbabe-5611-4016-bb56-0c00aa4906da"} 2026-07-02T03:14:53.884Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "159b25c7-6af9-4fc5-bbe5-5d146f36427d", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:14:55.870Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "30fec721-24f9-4b26-9a0b-0bd76902f66f", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:14:57.247Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "159b25c7-6af9-4fc5-bbe5-5d146f36427d"} 2026-07-02T03:14:59.277Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "f68bee6b-3d26-4987-8269-62d6bf4772c1", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:15:00.885Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "fb9b9408-8247-45bd-a055-2bbaec1e112e", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:15:02.655Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "f68bee6b-3d26-4987-8269-62d6bf4772c1"} 2026-07-02T03:15:04.676Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "42d17180-0633-46b0-b5ab-52da03f2137d", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:15:05.945Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "fd703705-6b3d-4441-b5b8-37249aaf00da", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:15:08.769Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "42d17180-0633-46b0-b5ab-52da03f2137d"} 2026-07-02T03:15:10.883Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "a2f5b816-958d-4ab6-a1ef-80e96d686d8b", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:15:10.980Z INFO Waiting for prepare job to finish {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "7d42c6a0-a15c-444e-8634-8bc6a54b7466", "job": "prepare-job-backup-minio-cross-site-source"} 2026-07-02T03:15:14.147Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "a2f5b816-958d-4ab6-a1ef-80e96d686d8b"} 2026-07-02T03:15:16.011Z INFO starting cluster {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "0807180e-db84-4db3-8c18-2bd9dec01b8e", "cluster": "cross-site-source"} 2026-07-02T03:15:16.209Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e667c1f7-b9f1-4cfd-88a3-adf268a210a4", "object": "cross-site-source-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:15:16.264Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e667c1f7-b9f1-4cfd-88a3-adf268a210a4", "object": "cross-site-source-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:15:16.344Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e667c1f7-b9f1-4cfd-88a3-adf268a210a4", "object": "cross-site-source-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:15:17.366Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "9ab93153-1c78-49ca-820d-e4bbd66493ad", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:15:21.103Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "5a317db9-10bb-4b44-8aa0-373bbc95557b", "cluster": "cross-site-source"} 2026-07-02T03:15:21.227Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "9ab93153-1c78-49ca-820d-e4bbd66493ad"} 2026-07-02T03:15:22.713Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "3a1fd8ef-c324-4e3d-9686-3d61ac83f11f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:15:26.128Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "b0251e84-c1e8-4f5d-9abf-54d6afa7cc3c", "cluster": "cross-site-source"} 2026-07-02T03:15:26.158Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "3a1fd8ef-c324-4e3d-9686-3d61ac83f11f"} 2026-07-02T03:15:28.048Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "29b3e35a-1b30-4ddc-9f6f-f44558520f36", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:15:31.146Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "29b3e35a-1b30-4ddc-9f6f-f44558520f36"} 2026-07-02T03:15:31.146Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "bd767182-b78d-4bd2-8efe-8044b998267b", "cluster": "cross-site-source"} 2026-07-02T03:15:34.460Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "69b9628d-df24-4c1f-9a0e-cdc3432c754d", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:15:36.182Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "d4d5b69f-538d-4a75-b6d9-3b0f6579ded8", "cluster": "cross-site-source"} 2026-07-02T03:15:36.864Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "69b9628d-df24-4c1f-9a0e-cdc3432c754d"} 2026-07-02T03:15:40.580Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "134c5959-4302-40b2-81a9-3cad130e18d4", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:15:41.198Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "34f2f7b0-50da-463e-bc98-94125c370075", "cluster": "cross-site-source"} 2026-07-02T03:15:43.756Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "134c5959-4302-40b2-81a9-3cad130e18d4"} 2026-07-02T03:15:45.983Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b67c8976-2046-417a-b90a-dea27755b945", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:15:46.215Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "5cb09c95-f203-4f0a-8aab-4903be403f3a", "cluster": "cross-site-source"} 2026-07-02T03:15:49.449Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b67c8976-2046-417a-b90a-dea27755b945"} 2026-07-02T03:15:51.235Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "69eda960-4b04-4315-a414-5d144fd23084", "cluster": "cross-site-source"} 2026-07-02T03:15:51.372Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c6c9cebf-e0ea-4e96-adac-b358fe743308", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:15:54.982Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c6c9cebf-e0ea-4e96-adac-b358fe743308"} 2026-07-02T03:15:56.249Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "9116e745-68ac-4a49-9e12-895a41d077f6", "cluster": "cross-site-source"} 2026-07-02T03:15:56.758Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "daf813a7-22e3-43a9-819e-9822c5ee52ee", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:16:00.445Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "daf813a7-22e3-43a9-819e-9822c5ee52ee"} 2026-07-02T03:16:01.266Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "5b20bb49-73b0-4bb1-934f-ebc95f429bab", "cluster": "cross-site-source"} 2026-07-02T03:16:02.076Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "86a82cd2-699d-409d-8031-04aa1603c207", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:16:05.473Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "86a82cd2-699d-409d-8031-04aa1603c207"} 2026-07-02T03:16:06.280Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "96f372be-331b-40c1-a400-3d6fbb26728a", "cluster": "cross-site-source"} 2026-07-02T03:16:07.476Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "aa4e13a0-9b9a-4af9-bd23-3ff6391b9921", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:16:10.966Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "aa4e13a0-9b9a-4af9-bd23-3ff6391b9921"} 2026-07-02T03:16:11.293Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "a42c5c2c-70b2-4632-88e5-463359420a9c", "cluster": "cross-site-source"} 2026-07-02T03:16:12.855Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "f48936ee-96d0-44df-8073-374c44417a04", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:16:16.309Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "4397e256-2a75-46a8-a123-6bf1df7a721a", "cluster": "cross-site-source"} 2026-07-02T03:16:16.641Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "f48936ee-96d0-44df-8073-374c44417a04"} 2026-07-02T03:16:18.204Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "0b9a9a2a-a0a9-4bd7-be63-33cc08f24d4a", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:16:21.329Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "903c0e06-8b45-420c-b67a-eebe65b65a2a", "cluster": "cross-site-source"} 2026-07-02T03:16:22.580Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "0b9a9a2a-a0a9-4bd7-be63-33cc08f24d4a"} 2026-07-02T03:16:23.558Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c39da930-3dc4-4a9a-a516-ad99aa111f28", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:16:26.344Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "b9b60d52-1478-4dbb-ba3d-243b1f2fe2cd", "cluster": "cross-site-source"} 2026-07-02T03:16:27.279Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c39da930-3dc4-4a9a-a516-ad99aa111f28"} 2026-07-02T03:16:28.878Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "edd8c7ec-f85b-4a4d-ae34-d4392dbe0521", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:16:31.363Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "61759155-f2ec-42bd-b223-ec5d1b504abf", "cluster": "cross-site-source"} 2026-07-02T03:16:32.031Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "edd8c7ec-f85b-4a4d-ae34-d4392dbe0521"} 2026-07-02T03:16:34.259Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "7a2058ba-e693-42d3-9a0f-20c0abd4aba5", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:16:36.376Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "80a0ae6c-32cf-4a62-8afc-6c8a33339dba", "cluster": "cross-site-source"} 2026-07-02T03:16:37.562Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "7a2058ba-e693-42d3-9a0f-20c0abd4aba5"} 2026-07-02T03:16:39.659Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "1ccc7252-1192-418c-9d5f-a4bee45e2090", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:16:41.402Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "105e921f-1983-48b8-af30-dbab6a44c5fc", "cluster": "cross-site-source"} 2026-07-02T03:16:42.770Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "1ccc7252-1192-418c-9d5f-a4bee45e2090"} 2026-07-02T03:16:45.202Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "a4a55568-481b-4073-bd13-f3aca75a27d6", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:16:46.416Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "65e7efe8-48f4-4c27-93b7-9261014db0cf", "cluster": "cross-site-source"} 2026-07-02T03:16:48.940Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "a4a55568-481b-4073-bd13-f3aca75a27d6"} 2026-07-02T03:16:50.590Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "cd2b3871-50b1-40d9-8ffc-888196456b4d", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:16:51.434Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "0c27c4db-ea13-4513-ac29-bc171c9de53c", "cluster": "cross-site-source"} 2026-07-02T03:16:53.979Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "cd2b3871-50b1-40d9-8ffc-888196456b4d"} 2026-07-02T03:16:55.962Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "1e3b4509-18c4-4a05-968a-c2d52ac94605", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:16:56.452Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "c541ff16-89d5-494a-abba-eee19afb6151", "cluster": "cross-site-source"} 2026-07-02T03:16:59.275Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "1e3b4509-18c4-4a05-968a-c2d52ac94605"} 2026-07-02T03:17:01.277Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "63648b0f-070d-4f68-86af-0d837a4c3ed6", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:17:01.469Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "bbc1acc2-27bb-441f-bf4a-50dcb76daf9f", "cluster": "cross-site-source"} 2026-07-02T03:17:04.857Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "63648b0f-070d-4f68-86af-0d837a4c3ed6"} 2026-07-02T03:17:06.496Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "72f94523-cf31-49d3-ba0e-cecb23b86d4f", "cluster": "cross-site-source"} 2026-07-02T03:17:06.748Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "846fa110-5c9b-4bab-8504-3dc951a83d3c", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:17:10.352Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "846fa110-5c9b-4bab-8504-3dc951a83d3c"} 2026-07-02T03:17:11.515Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "762e676b-e068-4401-b471-9ec30457875b", "cluster": "cross-site-source"} 2026-07-02T03:17:12.165Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b11e3cea-c62b-4bac-8222-8e13a93b33cb", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:17:15.534Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b11e3cea-c62b-4bac-8222-8e13a93b33cb"} 2026-07-02T03:17:16.534Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "76be6d8a-284d-465c-b974-b4dbb7b099eb", "cluster": "cross-site-source"} 2026-07-02T03:17:17.581Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "7de87438-4c96-4ccb-b7b2-f42afaca54ba", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:17:21.272Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "7de87438-4c96-4ccb-b7b2-f42afaca54ba"} 2026-07-02T03:17:21.550Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "50e90a60-c0dc-4c29-a05a-551303824225", "cluster": "cross-site-source"} 2026-07-02T03:17:23.066Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "4d7b8af6-0b13-4510-b49b-69e42a6fa4a4", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:17:26.567Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "19674496-161c-429e-97b2-950816de0d3d", "cluster": "cross-site-source"} 2026-07-02T03:17:26.644Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "4d7b8af6-0b13-4510-b49b-69e42a6fa4a4"} 2026-07-02T03:17:28.479Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "83a904e0-00b1-464e-998d-affb72589b44", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:17:31.584Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "cd783ad9-8d2a-4b23-856c-29031ce23ff1", "cluster": "cross-site-source"} 2026-07-02T03:17:31.779Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "83a904e0-00b1-464e-998d-affb72589b44"} 2026-07-02T03:17:33.881Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "aa28920f-e4ed-4406-98f0-ef179da192d1", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:17:36.598Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "0e7b1bdf-e959-483e-9ce9-830ba669a95c", "cluster": "cross-site-source"} 2026-07-02T03:17:37.545Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "aa28920f-e4ed-4406-98f0-ef179da192d1"} 2026-07-02T03:17:39.260Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "bb2a9ed3-e4e3-4c1a-8aa5-21c4d8c66743", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:17:41.613Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "8e35ea36-76b7-4e2c-9a48-2df020edfc53", "cluster": "cross-site-source"} 2026-07-02T03:17:42.878Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "bb2a9ed3-e4e3-4c1a-8aa5-21c4d8c66743"} 2026-07-02T03:17:44.758Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "f24cd296-faeb-443c-8a0f-25e8448e9db0", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:17:46.627Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "ca3c792c-6ed3-4411-8737-cc04e42c74e2", "cluster": "cross-site-source"} 2026-07-02T03:17:48.280Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "f24cd296-faeb-443c-8a0f-25e8448e9db0"} 2026-07-02T03:17:50.170Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "fe750a6e-ef3b-4b8c-affc-42bfc5a863fd", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:17:51.640Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "1caed541-363a-4910-81a9-828560f348a1", "cluster": "cross-site-source"} 2026-07-02T03:17:54.039Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "fe750a6e-ef3b-4b8c-affc-42bfc5a863fd"} 2026-07-02T03:17:55.680Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "23a4ea26-8d34-4b61-88dc-847f172c6a3f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 6/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:17:56.658Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "f0a9854b-2202-4ef5-bac0-32d9b8efb8c0", "cluster": "cross-site-source"} 2026-07-02T03:17:59.379Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "23a4ea26-8d34-4b61-88dc-847f172c6a3f"} 2026-07-02T03:18:01.061Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8c6113fe-dbad-471c-b6c9-c8b1308f02f4", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 6/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:18:01.674Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "27c27bf0-f3c4-4af3-9c03-eebf0ef67387", "cluster": "cross-site-source"} 2026-07-02T03:18:04.873Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "8c6113fe-dbad-471c-b6c9-c8b1308f02f4"} 2026-07-02T03:18:06.444Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "af5a0f4e-1016-4e24-a6c9-2caecfc7791c", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 6/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:18:06.688Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "c64cb40f-e4b4-4e5a-8216-77faba4092e3", "cluster": "cross-site-source"} 2026-07-02T03:18:10.124Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "af5a0f4e-1016-4e24-a6c9-2caecfc7791c"} 2026-07-02T03:18:11.707Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "6c286f42-3751-4f73-a579-a51e557da28c", "cluster": "cross-site-source"} 2026-07-02T03:18:11.858Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "1a5d1ab7-8e44-442e-902f-3f3e9d8fb7e8", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 6/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:18:15.244Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "1a5d1ab7-8e44-442e-902f-3f3e9d8fb7e8"} 2026-07-02T03:18:16.725Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "003cde35-df5d-4786-b5c0-3672cc61546b", "cluster": "cross-site-source"} 2026-07-02T03:18:17.220Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "d91f67f8-4ee9-4a12-a4cc-a5c79e304185", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 6/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:18:21.162Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "d91f67f8-4ee9-4a12-a4cc-a5c79e304185"} 2026-07-02T03:18:21.741Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "4019b3ca-713b-4f14-8538-50579b35e358", "cluster": "cross-site-source"} 2026-07-02T03:18:22.673Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "fe0d3dc7-13af-4197-93f0-e8a2a47b3a16", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 6/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:18:26.262Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "fe0d3dc7-13af-4197-93f0-e8a2a47b3a16"} 2026-07-02T03:18:26.776Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "30b0a771-8a1f-49d5-8a58-1684a19fbbde", "cluster": "cross-site-source"} 2026-07-02T03:18:28.226Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "4dbb60eb-a13b-45ee-8285-dd7c128e1412", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 6/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:18:31.380Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "4dbb60eb-a13b-45ee-8285-dd7c128e1412"} 2026-07-02T03:18:31.806Z INFO Waiting for cluster to start {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "PerconaXtraDBClusterRestore": {"name":"backup-minio","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "backup-minio", "reconcileID": "6f13187f-ffaf-4b01-a9b4-200f389b8852", "cluster": "cross-site-source"} 2026-07-02T03:18:33.665Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "3348a4bd-e68b-46f0-adc5-0205bf9917ea", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 6/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:18:37.432Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "3348a4bd-e68b-46f0-adc5-0205bf9917ea"} 2026-07-02T03:18:39.018Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "f10a1584-4e8a-42ec-b90d-d781e9755c82", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 6/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:18:42.358Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "f10a1584-4e8a-42ec-b90d-d781e9755c82"} 2026-07-02T03:18:44.377Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "2117e9c3-8754-40a7-b42f-d162b340ab29", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 6/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:18:48.157Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "2117e9c3-8754-40a7-b42f-d162b340ab29"} 2026-07-02T03:18:49.846Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c69c2f52-b81e-47aa-a85f-39e932d67cfe", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@34.118.228.242:3306'. This was attempt 6/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '34.118.228.242:3306' (111)"} 2026-07-02T03:18:54.074Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c69c2f52-b81e-47aa-a85f-39e932d67cfe"} 2026-07-02T03:18:59.034Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ca73a6b7-d941-4c34-bfd5-25ce9a49ab63"} 2026-07-02T03:19:04.177Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "ba40858b-decb-4df6-b892-943f19fdf79f"} 2026-07-02T03:19:09.545Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "857d96bc-d75e-4b7c-8a00-5f0da736b77e"} 2026-07-02T03:19:10.267Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "be28965e-d119-4965-b298-529f30929686", "object": "cross-site-replica-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:19:10.319Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "be28965e-d119-4965-b298-529f30929686", "object": "cross-site-replica-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:19:10.405Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "be28965e-d119-4965-b298-529f30929686", "err": "get primary pxc pod: not found"} 2026-07-02T03:19:13.665Z ERROR sync users {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "be28965e-d119-4965-b298-529f30929686", "error": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") / / ", "errorVerbose": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") / / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:1000\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:859\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T03:19:16.192Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "93aab786-a9e6-4acd-9bf4-10d6e774648a", "err": "get primary pxc pod: not found"} 2026-07-02T03:19:16.400Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "43265d47-a4e3-4d2e-a9b0-272f782d1dc1", "err": "get primary pxc pod: not found"} 2026-07-02T03:19:18.461Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "c4862fbd-31f2-43f8-aa0a-c92d0d37de71", "err": "get primary pxc pod: not found"} 2026-07-02T03:19:20.299Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "a7545b8f-2d07-4a7a-9a64-90dea8bfa00d"} 2026-07-02T03:19:26.188Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "cb08138f-bb4b-4c66-9802-dc399f10af59"} 2026-07-02T03:19:29.154Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "1c54b7ec-fa02-4f7d-a972-904d33a5a01f", "err": "get primary pxc pod: not found"} 2026-07-02T03:19:33.035Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "a33a1126-c5ee-402d-85bf-2bbf09ad7724"} 2026-07-02T03:19:50.391Z INFO Remove outdated replication channel {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "973db75c-1ebe-41de-b0a6-ff53359958a0", "channel": "source_to_replica"} 2026-07-02T03:19:50.393Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "973db75c-1ebe-41de-b0a6-ff53359958a0", "channel": "source_to_replica", "host": "34.118.228.242"} 2026-07-02T03:19:50.393Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "973db75c-1ebe-41de-b0a6-ff53359958a0", "channel": "source_to_replica", "host": "34.118.235.139"} 2026-07-02T03:19:50.394Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "973db75c-1ebe-41de-b0a6-ff53359958a0", "channel": "source_to_replica", "host": "34.118.239.56"} 2026-07-02T03:19:50.397Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "973db75c-1ebe-41de-b0a6-ff53359958a0", "pod": "cross-site-replica-pxc-0"} 2026-07-02T03:19:50.401Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "973db75c-1ebe-41de-b0a6-ff53359958a0", "pod": "cross-site-replica-pxc-1"} 2026-07-02T03:19:50.406Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "973db75c-1ebe-41de-b0a6-ff53359958a0", "pod": "cross-site-replica-pxc-2"} 2026-07-02T03:19:51.533Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "bba6a19a-6d5d-4336-8243-ebdc37f86452", "err": "get primary pxc pod: not found"} 2026-07-02T03:19:54.991Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "96b07eb4-b2f8-4943-bad8-63c7a322ff32"} 2026-07-02T03:20:00.258Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "296a7b13-e737-485d-a66e-995d3435eac8"} 2026-07-02T03:20:05.459Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "82c2403e-b3e7-4512-a507-5b720e1b5f8d"} 2026-07-02T03:20:10.477Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "c4525220-ed35-46bb-84ef-76da80c873e9"} 2026-07-02T03:20:16.357Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "599fc84f-82eb-49bb-858d-9b7e3ea71e27"} 2026-07-02T03:20:21.255Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "c9c0fcd8-0a95-47fc-b780-920ffe826a4b"} 2026-07-02T03:20:26.266Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "844f7709-3d1a-47f7-a37b-5f35177beb89"} 2026-07-02T03:20:31.783Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "66fecc71-4519-4374-a5bb-8db8cb1ae0a9"} 2026-07-02T03:20:37.155Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "fd64fe2d-4af5-4a59-86a3-5f0e6676406a"} 2026-07-02T03:20:42.280Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "3e732ca5-61c6-494a-9e7f-58e3c2fff43b"} 2026-07-02T03:20:47.283Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "c0d66d66-e311-412d-ae16-ea02fa698480"} 2026-07-02T03:20:52.887Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "77921943-851a-4d08-8bb1-ad40e3ba481d"} 2026-07-02T03:20:57.935Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "c5441111-2cdb-4a6c-8bc5-b310109474c5"} 2026-07-02T03:21:03.148Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "8e3b8df0-c358-48ba-9a78-d056fbd0444e"} 2026-07-02T03:21:08.952Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "1bf796c7-2f32-41d6-a086-3e978cb08c19"} 2026-07-02T03:21:14.173Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "a31774c1-6805-4268-b05e-167f0afd1664"} 2026-07-02T03:21:19.484Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "d860332a-90cd-48a8-89e5-6830946fa12d"} 2026-07-02T03:21:24.780Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "5dd0910c-ebcd-46bb-8aa5-19c4ec6979f8"} 2026-07-02T03:21:29.968Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "bc77716f-7683-4588-abcf-acc23d5c61d9"} 2026-07-02T03:21:35.239Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "de9954b4-ea66-4113-a60d-04fc2a0d5894"} 2026-07-02T03:21:40.358Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "837d97d8-f9b9-405e-8c69-f7d90d3c64e5"} 2026-07-02T03:21:45.375Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "af7e994d-6e44-431c-853c-2557bc3bb866"} 2026-07-02T03:21:51.137Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "c73dbe38-cfa6-45e1-bad1-7a918fc2fcfb"} 2026-07-02T03:21:56.186Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "e9427c50-b3f5-44d2-87f7-63324b9f12d3"} 2026-07-02T03:22:01.568Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "4d2bd837-ddba-4972-b570-1a4584524ca7"} 2026-07-02T03:22:06.480Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "6ef713ab-ea8d-40c8-bb46-e9cb8850082a"} 2026-07-02T03:22:11.746Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "5ccb6fde-7988-4144-9f80-b3d5651471b8"} 2026-07-02T03:22:13.195Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "9d4556c2-4130-4b2a-ac2d-0fef727a126e", "object": "cross-site-source-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:22:13.248Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "9d4556c2-4130-4b2a-ac2d-0fef727a126e", "object": "cross-site-source-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T03:22:13.347Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "9d4556c2-4130-4b2a-ac2d-0fef727a126e", "pod": "cross-site-source-pxc-1"} 2026-07-02T03:22:13.358Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "9d4556c2-4130-4b2a-ac2d-0fef727a126e", "pod": "cross-site-source-pxc-2"} 2026-07-02T03:22:13.368Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "9d4556c2-4130-4b2a-ac2d-0fef727a126e", "pod": "cross-site-source-pxc-0"} 2026-07-02T03:22:13.499Z INFO Replication pod has changed {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "9d4556c2-4130-4b2a-ac2d-0fef727a126e", "new replication pod": "cross-site-source-pxc-0"} 2026-07-02T03:22:17.171Z ERROR sync users {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "9d4556c2-4130-4b2a-ac2d-0fef727a126e", "error": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") / / ", "errorVerbose": "exec syncusers: failed to execute command in pod: unable to upgrade connection: container not found (\"proxysql\") / / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:1000\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:859\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-07-02T03:22:26.114Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "36036dd4-0c91-4412-881a-0d8a26163cbb"} 2026-07-02T03:22:30.982Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "509d6906-6e70-4039-a644-2afd231610b5"} 2026-07-02T03:22:36.675Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "52b45078-9fe0-4f5e-ab4a-cd97f4346cf7"} 2026-07-02T03:22:42.412Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "4a08c955-6f3c-479b-afc2-8516295d0a14"} 2026-07-02T03:22:47.480Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "74068bea-6af2-4344-85d9-4630cd9400e6"} 2026-07-02T03:22:59.880Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "6162a261-b8e9-4f07-8835-3f56396a7541"} 2026-07-02T03:23:05.191Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "793aa3f6-28e9-44f4-8141-e6dab2babe02"} 2026-07-02T03:23:10.669Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "e4b70d51-729a-481e-9165-50d0fe402ac5"} 2026-07-02T03:23:15.596Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "49a41061-fb34-451e-8443-9c0e1567ff58"} 2026-07-02T03:23:21.684Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "b2ffbcaf-66d7-40ba-a167-18dce53ad9aa"} 2026-07-02T03:23:26.183Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "a78c3ad5-83ec-4675-8440-5fc754f6a241"} 2026-07-02T03:23:31.695Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "10e86473-2e78-48db-90ba-39c07e525adc"} 2026-07-02T03:23:37.373Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "af9d36c5-6cff-4594-8b27-b342f5f49157"} 2026-07-02T03:23:41.993Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "428f68ac-a1f5-4d2d-bdee-10f55a3328a2"} 2026-07-02T03:23:47.198Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-replica","namespace":"cross-site-proxysql-replica-17697"}, "namespace": "cross-site-proxysql-replica-17697", "name": "cross-site-replica", "reconcileID": "45797503-8115-4d55-8bd4-3d304eebd84b"} 2026-07-02T03:23:50.635Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "7657df34-b797-4f72-8bb0-c42308f4c911"} 2026-07-02T03:23:56.330Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"cross-site-source","namespace":"cross-site-proxysql-28035"}, "namespace": "cross-site-proxysql-28035", "name": "cross-site-source", "reconcileID": "91cdd1b3-b78a-42fd-8a92-93350d1d1f3f"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:861 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 -  }, -  { -  }, +  }, -  Annotations: map[string]string{ +  Annotations: map[string]string{ -  APIVersion: "apps/v1", -  APIVersion: "apps/v1", +  AvailableReplicas: 0, -  AvailableReplicas: 3, -  CollisionCount: &0, +  CollisionCount: nil, +  CreationTimestamp: v1.Time{}, -  CreationTimestamp: v1.Time{Time: s"2026-07-02 02:43:18 +0000 UTC"}, -  CreationTimestamp: v1.Time{Time: s"2026-07-02 02:53:47 +0000 UTC"}, +  CurrentReplicas: 0, -  CurrentReplicas: 3, +  CurrentRevision: "", -  CurrentRevision: "cross-site-replica-proxysql-577966cf8b", -  CurrentRevision: "cross-site-replica-proxysql-67874c6964", -  CurrentRevision: "cross-site-replica-pxc-8674f56b59", -  CurrentRevision: "cross-site-source-proxysql-664db96fdb", -  CurrentRevision: "cross-site-source-pxc-6988c69dbf", -  DefaultMode: &420, -  DefaultMode: &420, +  DefaultMode: nil, +  DefaultMode: nil, +  DeprecatedServiceAccount: "", -  DeprecatedServiceAccount: "default", +  DNSPolicy: "", -  DNSPolicy: "ClusterFirst", -  FieldsType: "FieldsV1", -  FieldsType: "FieldsV1", -  FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., -  FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., +  Generation: 0, -  Generation: 1, -  Generation: 2, -  Generation: 3, -  Generation: 4, +  ManagedFields: nil, -  ManagedFields: []v1.ManagedFieldsEntry{ -  Manager: "kube-controller-manager", -  Manager: "percona-xtradb-cluster-operator", +  ObservedGeneration: 0, -  ObservedGeneration: 1, -  ObservedGeneration: 2, -  ObservedGeneration: 3, -  ObservedGeneration: 4, -  Operation: "Update", -  Operation: "Update", -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJjcm9zcy1zaXRlLXJlcGxpY2EiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIs"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJjcm9zcy1zaXRlLXJlcGxpY2EiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIs"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJjcm9zcy1zaXRlLXNvdXJjZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwi"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJjcm9zcy1zaXRlLXNvdXJjZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwi"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtc291cmNlIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtc291cmNlIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtcmVwbGljYSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJh"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6MCwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtcmVwbGljYSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJh"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJjcm9zcy1zaXRlLXJlcGxpY2EiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIs"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJjcm9zcy1zaXRlLXJlcGxpY2EiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIs"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJjcm9zcy1zaXRlLXNvdXJjZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwi"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJjcm9zcy1zaXRlLXNvdXJjZSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwi"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtc291cmNlIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtc291cmNlIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtc291cmNlIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtc291cmNlIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InByb3h5c3FsIiwicGVyY29uYS5jb20vY29uZmlndXJhdGlvbi1oYXNoIjoiZDQxZDhjZDk4ZjAwYjIwNGU5ODAwOTk4ZWNmODQyN2UiLCJwZXJjb25hLmNvbS9zc2wtaGFzaCI6ImU3YTVjMWIxOWE2YjYzZjBiYzZlNDI1ZTliNWVjMjFjIiwicGVyY29uYS5jb20vc3NsLWludGVybmFsLWhhc2giOiJhZjE0MzhjMjNhYThhYzVlYWFkYjgwNDIyYjdiOGI0MSJ9fSwic3BlYyI6eyJ2b2x1bWVzIjpbeyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJjcm9zcy1zaXRlLXNvdXJjZS1zc2wtaW50ZXJuYWwiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6InNzbCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiY3Jvc3Mtc2l0ZS1zb3VyY2Utc3NsIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoiYmluIiwiZW1wdHlEaXIiOnt9fV0sImluaXRDb250YWluZXJzIjpbeyJuYW1lIjoicHhjLWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTM5LWRjZGYxZThhIiwiY29tbWFuZCI6WyIvcHhjLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiYmluIiwibW91bnRQYXRoIjoiL3Zhci9saWIvbXlzcWwifV0sImltYWdlUHVsbFBvbGljeSI6IkFsd2F5cyJ9LHsibmFtZSI6InByb3h5c3FsLWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTM5LWRjZGYxZThhIiwiY29tbWFuZCI6WyIvcHJveHlzcWwtaW5pdC1lbnRyeXBvaW50LnNoIl0sInJlc291cmNlcyI6eyJsaW1pdHMiOnsiY3B1IjoiNTBtIiwibWVtb3J5IjoiNTBNIn19LCJ2b2x1bWVNb3VudHMiOlt7Im5hbWUiOiJiaW4iLCJtb3VudFBhdGgiOiIvb3B0L3BlcmNvbmEifV0sImltYWdlUHVsbFBvbGljeSI6IkFsd2F5cyJ9XSwiY29udGFpbmVycyI6W3sibmFtZSI6InByb3h5c3FsIiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6bWFpbi1wcm94eXNxbCIsImNvbW1hbmQiOlsiL29wdC9wZXJjb25hL3Byb3h5c3FsLWVudHJ5cG9pbnQuc2giXSwiYXJncyI6WyJwcm94eXNxbCIsIi1mIiwiLWMiLCIvZXRjL3Byb3h5c3FsL3Byb3h5c3FsLmNuZiIsIi0tcmVsb2FkIl0sInBvcnRzIjpbeyJuYW1lIjoibXlzcWwiLCJjb250YWluZXJQb3J0IjozMzA2fSx7Im5hbWUiOiJwcm94eWFkbSIsImNvbnRhaW5lclBvcnQiOjYwMzJ9LHsibmFtZSI6InN0YXRzIiwiY29udGFpbmVyUG9ydCI6NjA3MH1dLCJlbnZGcm9tIjpbeyJzZWNyZXRSZWYiOnsibmFtZSI6ImNyb3NzLXNpdGUtc291cmNlLWVudi12YXJzLXByb3h5c3FsIiwib3B0aW9uYWwiOnRydWV9fV0sImVudiI6W3sibmFtZSI6IlBYQ19TRVJWSUNFIiwidmFsdWUiOiJjcm9zcy1zaXRlLXNvdXJjZS1weGMifSx7Im5hbWUiOiJPUEVSQVRPUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLWNyb3NzLXNpdGUtc291cmNlIiwia2V5Ijoib3BlcmF0b3IifX19LHsibmFtZSI6IlBST1hZX0FETUlOX1VTRVIiLCJ2YWx1ZSI6InByb3h5YWRtaW4ifSx7Im5hbWUiOiJQUk9YWV9BRE1JTl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLWNyb3NzLXNpdGUtc291cmNlIiwia2V5IjoicHJveHlhZG1pbiJ9fX0seyJuYW1lIjoiTU9OSVRPUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLWNyb3NzLXNpdGUtc291cmNlIiwia2V5IjoibW9uaXRvciJ9fX0seyJuYW1lIjoiU0NIRURVTEVSX0NIRUNLVElNRU9VVCIsInZhbHVlIjoiMjAwMCJ9LHsibmFtZSI6IlNDSEVEVUxFUl9XUklURVJBTFNPUkVBREVSIiwidmFsdWUiOiIxIn0seyJuYW1lIjoiU0NIRURVTEVSX1JFVFJZVVAiLCJ2YWx1ZSI6IjEifSx7Im5hbWUiOiJTQ0hFRFVMRVJfUkVUUllET1dOIiwidmFsdWUiOiIzIn0seyJuYW1lIjoiU0NIRURVTEVSX1BJTkdUSU1FT1VUIiwidmFsdWUiOiIxMDAwIn0seyJuYW1lIjoiU0NIRURVTEVSX05PREVDSEVDS0lOVEVSVkFMIiwidmFsdWUiOiIyMDAwIn0seyJuYW1lIjoiU0NIRURVTEVSX01BWENPTk5FQ1RJT05TIiwidmFsdWUiOiIxMDAwIn0seyJuYW1lIjoiUEVSQ09OQV9TQ0hFRFVMRVJfQ0ZHIiwidmFsdWUiOiIvdG1wL3NjaGVkdWxlci1jb25maWcudG9tbCJ9LHsibmFtZSI6IlNDSEVEVUxFUl9FTkFCTEVEIiwidmFsdWUiOiJ0cnVlIn0seyJuYW1lIjoiUFhDX1JFQURfT05MWSIsInZhbHVlIjoidHJ1ZSJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI3MDBtIiwibWVtb3J5Ijoi"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtc291cmNlIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn19LCJ0ZW1wbGF0ZSI6eyJtZXRhZGF0YSI6eyJsYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtc291cmNlIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJuZXRlcy5pby9uYW1lIjoicGVyY29uYS14dHJhZGItY2x1c3RlciIsImFwcC5rdWJlcm5ldGVzLmlvL3BhcnQtb2YiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIn0sImFubm90YXRpb25zIjp7Imt1YmVjdGwua3ViZXJuZXRlcy5pby9kZWZhdWx0LWNvbnRhaW5lciI6InByb3h5c3FsIiwicGVyY29uYS5jb20vY29uZmlndXJhdGlvbi1oYXNoIjoiZDQxZDhjZDk4ZjAwYjIwNGU5ODAwOTk4ZWNmODQyN2UiLCJwZXJjb25hLmNvbS9zc2wtaGFzaCI6ImU3YTVjMWIxOWE2YjYzZjBiYzZlNDI1ZTliNWVjMjFjIiwicGVyY29uYS5jb20vc3NsLWludGVybmFsLWhhc2giOiJhZjE0MzhjMjNhYThhYzVlYWFkYjgwNDIyYjdiOGI0MSJ9fSwic3BlYyI6eyJ2b2x1bWVzIjpbeyJuYW1lIjoic3NsLWludGVybmFsIiwic2VjcmV0Ijp7InNlY3JldE5hbWUiOiJjcm9zcy1zaXRlLXNvdXJjZS1zc2wtaW50ZXJuYWwiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6InNzbCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiY3Jvc3Mtc2l0ZS1zb3VyY2Utc3NsIiwib3B0aW9uYWwiOmZhbHNlfX0seyJuYW1lIjoiYmluIiwiZW1wdHlEaXIiOnt9fV0sImluaXRDb250YWluZXJzIjpbeyJuYW1lIjoicHhjLWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTM5LWRjZGYxZThhIiwiY29tbWFuZCI6WyIvcHhjLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiYmluIiwibW91bnRQYXRoIjoiL3Zhci9saWIvbXlzcWwifV0sImltYWdlUHVsbFBvbGljeSI6IkFsd2F5cyJ9LHsibmFtZSI6InByb3h5c3FsLWluaXQiLCJpbWFnZSI6InBlcmNvbmFsYWIvcGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvcjpQUi0yNTM5LWRjZGYxZThhIiwiY29tbWFuZCI6WyIvcHJveHlzcWwtaW5pdC1lbnRyeXBvaW50LnNoIl0sInJlc291cmNlcyI6eyJsaW1pdHMiOnsiY3B1IjoiNTBtIiwibWVtb3J5IjoiNTBNIn19LCJ2b2x1bWVNb3VudHMiOlt7Im5hbWUiOiJiaW4iLCJtb3VudFBhdGgiOiIvb3B0L3BlcmNvbmEifV0sImltYWdlUHVsbFBvbGljeSI6IkFsd2F5cyJ9XSwiY29udGFpbmVycyI6W3sibmFtZSI6InByb3h5c3FsIiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6bWFpbi1wcm94eXNxbCIsImNvbW1hbmQiOlsiL29wdC9wZXJjb25hL3Byb3h5c3FsLWVudHJ5cG9pbnQuc2giXSwiYXJncyI6WyJwcm94eXNxbCIsIi1mIiwiLWMiLCIvZXRjL3Byb3h5c3FsL3Byb3h5c3FsLmNuZiIsIi0tcmVsb2FkIl0sInBvcnRzIjpbeyJuYW1lIjoibXlzcWwiLCJjb250YWluZXJQb3J0IjozMzA2fSx7Im5hbWUiOiJwcm94eWFkbSIsImNvbnRhaW5lclBvcnQiOjYwMzJ9LHsibmFtZSI6InN0YXRzIiwiY29udGFpbmVyUG9ydCI6NjA3MH1dLCJlbnZGcm9tIjpbeyJzZWNyZXRSZWYiOnsibmFtZSI6ImNyb3NzLXNpdGUtc291cmNlLWVudi12YXJzLXByb3h5c3FsIiwib3B0aW9uYWwiOnRydWV9fV0sImVudiI6W3sibmFtZSI6IlBYQ19TRVJWSUNFIiwidmFsdWUiOiJjcm9zcy1zaXRlLXNvdXJjZS1weGMifSx7Im5hbWUiOiJPUEVSQVRPUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLWNyb3NzLXNpdGUtc291cmNlIiwia2V5Ijoib3BlcmF0b3IifX19LHsibmFtZSI6IlBST1hZX0FETUlOX1VTRVIiLCJ2YWx1ZSI6InByb3h5YWRtaW4ifSx7Im5hbWUiOiJQUk9YWV9BRE1JTl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLWNyb3NzLXNpdGUtc291cmNlIiwia2V5IjoicHJveHlhZG1pbiJ9fX0seyJuYW1lIjoiTU9OSVRPUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLWNyb3NzLXNpdGUtc291cmNlIiwia2V5IjoibW9uaXRvciJ9fX0seyJuYW1lIjoiU0NIRURVTEVSX0NIRUNLVElNRU9VVCIsInZhbHVlIjoiMjAwMCJ9LHsibmFtZSI6IlNDSEVEVUxFUl9XUklURVJBTFNPUkVBREVSIiwidmFsdWUiOiIxIn0seyJuYW1lIjoiU0NIRURVTEVSX1JFVFJZVVAiLCJ2YWx1ZSI6IjEifSx7Im5hbWUiOiJTQ0hFRFVMRVJfUkVUUllET1dOIiwidmFsdWUiOiIzIn0seyJuYW1lIjoiU0NIRURVTEVSX1BJTkdUSU1FT1VUIiwidmFsdWUiOiIxMDAwIn0seyJuYW1lIjoiU0NIRURVTEVSX05PREVDSEVDS0lOVEVSVkFMIiwidmFsdWUiOiIyMDAwIn0seyJuYW1lIjoiU0NIRURVTEVSX01BWENPTk5FQ1RJT05TIiwidmFsdWUiOiIxMDAwIn0seyJuYW1lIjoiUEVSQ09OQV9TQ0hFRFVMRVJfQ0ZHIiwidmFsdWUiOiIvdG1wL3NjaGVkdWxlci1jb25maWcudG9tbCJ9LHsibmFtZSI6IlNDSEVEVUxFUl9FTkFCTEVEIiwidmFsdWUiOiJ0cnVlIn0seyJuYW1lIjoiUFhDX1JFQURfT05MWSIsInZhbHVlIjoiZmFsc2UifV0sInJlc291cmNlcyI6eyJsaW1pdHMiOnsiY3B1IjoiNzAwbSIsIm1lbW9yeSI6"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtcmVwbGljYSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJh"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtcmVwbGljYSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJh"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtcmVwbGljYSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6InByb3h5c3FsIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJjcm9zcy1zaXRlLXJlcGxpY2EiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoicHJveHlzcWwiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiOWVhMDViYWY3YWM0NjNmOTA4ZDM1YmFlY2JmZWVmMDQiLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6ImFmMTQzOGMyM2FhOGFjNWVhYWRiODA0MjJiN2I4YjQxIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJzc2wtaW50ZXJuYWwiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6ImNyb3NzLXNpdGUtcmVwbGljYS1zc2wtaW50ZXJuYWwiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6InNzbCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiY3Jvc3Mtc2l0ZS1yZXBsaWNhLXNzbCIsIm9wdGlvbmFsIjpmYWxzZX19LHsibmFtZSI6ImJpbiIsImVtcHR5RGlyIjp7fX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjUzOS1kY2RmMWU4YSIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifSx7Im5hbWUiOiJwcm94eXNxbC1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjUzOS1kY2RmMWU4YSIsImNvbW1hbmQiOlsiL3Byb3h5c3FsLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiYmluIiwibW91bnRQYXRoIjoiL29wdC9wZXJjb25hIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifV0sImNvbnRhaW5lcnMiOlt7Im5hbWUiOiJwcm94eXNxbCIsImltYWdlIjoicGVyY29uYWxhYi9wZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yOm1haW4tcHJveHlzcWwiLCJjb21tYW5kIjpbIi9vcHQvcGVyY29uYS9wcm94eXNxbC1lbnRyeXBvaW50LnNoIl0sImFyZ3MiOlsicHJveHlzcWwiLCItZiIsIi1jIiwiL2V0Yy9wcm94eXNxbC9wcm94eXNxbC5jbmYiLCItLXJlbG9hZCJdLCJwb3J0cyI6W3sibmFtZSI6Im15c3FsIiwiY29udGFpbmVyUG9ydCI6MzMwNn0seyJuYW1lIjoicHJveHlhZG0iLCJjb250YWluZXJQb3J0Ijo2MDMyfSx7Im5hbWUiOiJzdGF0cyIsImNvbnRhaW5lclBvcnQiOjYwNzB9XSwiZW52RnJvbSI6W3sic2VjcmV0UmVmIjp7Im5hbWUiOiJjcm9zcy1zaXRlLXJlcGxpY2EtZW52LXZhcnMtcHJveHlzcWwiLCJvcHRpb25hbCI6dHJ1ZX19XSwiZW52IjpbeyJuYW1lIjoiUFhDX1NFUlZJQ0UiLCJ2YWx1ZSI6ImNyb3NzLXNpdGUtcmVwbGljYS1weGMifSx7Im5hbWUiOiJPUEVSQVRPUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLWNyb3NzLXNpdGUtcmVwbGljYSIsImtleSI6Im9wZXJhdG9yIn19fSx7Im5hbWUiOiJQUk9YWV9BRE1JTl9VU0VSIiwidmFsdWUiOiJwcm94eWFkbWluIn0seyJuYW1lIjoiUFJPWFlfQURNSU5fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1jcm9zcy1zaXRlLXJlcGxpY2EiLCJrZXkiOiJwcm94eWFkbWluIn19fSx7Im5hbWUiOiJNT05JVE9SX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtY3Jvc3Mtc2l0ZS1yZXBsaWNhIiwia2V5IjoibW9uaXRvciJ9fX0seyJuYW1lIjoiU0NIRURVTEVSX0NIRUNLVElNRU9VVCIsInZhbHVlIjoiMjAwMCJ9LHsibmFtZSI6IlNDSEVEVUxFUl9XUklURVJBTFNPUkVBREVSIiwidmFsdWUiOiIxIn0seyJuYW1lIjoiU0NIRURVTEVSX1JFVFJZVVAiLCJ2YWx1ZSI6IjEifSx7Im5hbWUiOiJTQ0hFRFVMRVJfUkVUUllET1dOIiwidmFsdWUiOiIzIn0seyJuYW1lIjoiU0NIRURVTEVSX1BJTkdUSU1FT1VUIiwidmFsdWUiOiIxMDAwIn0seyJuYW1lIjoiU0NIRURVTEVSX05PREVDSEVDS0lOVEVSVkFMIiwidmFsdWUiOiIyMDAwIn0seyJuYW1lIjoiU0NIRURVTEVSX01BWENPTk5FQ1RJT05TIiwidmFsdWUiOiIxMDAwIn0seyJuYW1lIjoiUEVSQ09OQV9TQ0hFRFVMRVJfQ0ZHIiwidmFsdWUiOiIvdG1wL3NjaGVkdWxlci1jb25maWcudG9tbCJ9LHsibmFtZSI6IlNDSEVEVUxFUl9FTkFCTEVEIiwidmFsdWUiOiJ0cnVlIn0seyJuYW1lIjoiUFhDX1JFQURfT05MWSIsInZhbHVlIjoidHJ1ZSJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI3MDBtIiwi"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtcmVwbGljYSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6InByb3h5c3FsIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJjcm9zcy1zaXRlLXJlcGxpY2EiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoicHJveHlzcWwiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiOWVhMDViYWY3YWM0NjNmOTA4ZDM1YmFlY2JmZWVmMDQiLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6ImFmMTQzOGMyM2FhOGFjNWVhYWRiODA0MjJiN2I4YjQxIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJzc2wtaW50ZXJuYWwiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6ImNyb3NzLXNpdGUtcmVwbGljYS1zc2wtaW50ZXJuYWwiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6InNzbCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiY3Jvc3Mtc2l0ZS1yZXBsaWNhLXNzbCIsIm9wdGlvbmFsIjpmYWxzZX19LHsibmFtZSI6ImJpbiIsImVtcHR5RGlyIjp7fX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjUzOS1kY2RmMWU4YSIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifSx7Im5hbWUiOiJwcm94eXNxbC1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjUzOS1kY2RmMWU4YSIsImNvbW1hbmQiOlsiL3Byb3h5c3FsLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiYmluIiwibW91bnRQYXRoIjoiL29wdC9wZXJjb25hIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifV0sImNvbnRhaW5lcnMiOlt7Im5hbWUiOiJwcm94eXNxbCIsImltYWdlIjoicGVyY29uYWxhYi9wZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yOm1haW4tcHJveHlzcWwiLCJjb21tYW5kIjpbIi9vcHQvcGVyY29uYS9wcm94eXNxbC1lbnRyeXBvaW50LnNoIl0sImFyZ3MiOlsicHJveHlzcWwiLCItZiIsIi1jIiwiL2V0Yy9wcm94eXNxbC9wcm94eXNxbC5jbmYiLCItLXJlbG9hZCJdLCJwb3J0cyI6W3sibmFtZSI6Im15c3FsIiwiY29udGFpbmVyUG9ydCI6MzMwNn0seyJuYW1lIjoicHJveHlhZG0iLCJjb250YWluZXJQb3J0Ijo2MDMyfSx7Im5hbWUiOiJzdGF0cyIsImNvbnRhaW5lclBvcnQiOjYwNzB9XSwiZW52RnJvbSI6W3sic2VjcmV0UmVmIjp7Im5hbWUiOiJjcm9zcy1zaXRlLXJlcGxpY2EtZW52LXZhcnMtcHJveHlzcWwiLCJvcHRpb25hbCI6dHJ1ZX19XSwiZW52IjpbeyJuYW1lIjoiUFhDX1NFUlZJQ0UiLCJ2YWx1ZSI6ImNyb3NzLXNpdGUtcmVwbGljYS1weGMifSx7Im5hbWUiOiJPUEVSQVRPUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLWNyb3NzLXNpdGUtcmVwbGljYSIsImtleSI6Im9wZXJhdG9yIn19fSx7Im5hbWUiOiJQUk9YWV9BRE1JTl9VU0VSIiwidmFsdWUiOiJwcm94eWFkbWluIn0seyJuYW1lIjoiUFJPWFlfQURNSU5fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1jcm9zcy1zaXRlLXJlcGxpY2EiLCJrZXkiOiJwcm94eWFkbWluIn19fSx7Im5hbWUiOiJNT05JVE9SX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtY3Jvc3Mtc2l0ZS1yZXBsaWNhIiwia2V5IjoibW9uaXRvciJ9fX0seyJuYW1lIjoiU0NIRURVTEVSX0NIRUNLVElNRU9VVCIsInZhbHVlIjoiMjAwMCJ9LHsibmFtZSI6IlNDSEVEVUxFUl9XUklURVJBTFNPUkVBREVSIiwidmFsdWUiOiIxIn0seyJuYW1lIjoiU0NIRURVTEVSX1JFVFJZVVAiLCJ2YWx1ZSI6IjEifSx7Im5hbWUiOiJTQ0hFRFVMRVJfUkVUUllET1dOIiwidmFsdWUiOiIzIn0seyJuYW1lIjoiU0NIRURVTEVSX1BJTkdUSU1FT1VUIiwidmFsdWUiOiIxMDAwIn0seyJuYW1lIjoiU0NIRURVTEVSX05PREVDSEVDS0lOVEVSVkFMIiwidmFsdWUiOiIyMDAwIn0seyJuYW1lIjoiU0NIRURVTEVSX01BWENPTk5FQ1RJT05TIiwidmFsdWUiOiIxMDAwIn0seyJuYW1lIjoiUEVSQ09OQV9TQ0hFRFVMRVJfQ0ZHIiwidmFsdWUiOiIvdG1wL3NjaGVkdWxlci1jb25maWcudG9tbCJ9LHsibmFtZSI6IlNDSEVEVUxFUl9FTkFCTEVEIiwidmFsdWUiOiJ0cnVlIn0seyJuYW1lIjoiUFhDX1JFQURfT05MWSIsInZhbHVlIjoidHJ1ZSJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI3MDBtIiwi"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtcmVwbGljYSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6InByb3h5c3FsIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJjcm9zcy1zaXRlLXJlcGxpY2EiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoicHJveHlzcWwiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiOWVhMDViYWY3YWM0NjNmOTA4ZDM1YmFlY2JmZWVmMDQiLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6ImFmMTQzOGMyM2FhOGFjNWVhYWRiODA0MjJiN2I4YjQxIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJzc2wtaW50ZXJuYWwiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6ImNyb3NzLXNpdGUtcmVwbGljYS1zc2wtaW50ZXJuYWwiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6InNzbCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiY3Jvc3Mtc2l0ZS1yZXBsaWNhLXNzbCIsIm9wdGlvbmFsIjpmYWxzZX19LHsibmFtZSI6ImJpbiIsImVtcHR5RGlyIjp7fX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjUzOS1kY2RmMWU4YSIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifSx7Im5hbWUiOiJwcm94eXNxbC1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjUzOS1kY2RmMWU4YSIsImNvbW1hbmQiOlsiL3Byb3h5c3FsLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiYmluIiwibW91bnRQYXRoIjoiL29wdC9wZXJjb25hIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifV0sImNvbnRhaW5lcnMiOlt7Im5hbWUiOiJwcm94eXNxbCIsImltYWdlIjoicGVyY29uYWxhYi9wZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yOm1haW4tcHJveHlzcWwiLCJjb21tYW5kIjpbIi9vcHQvcGVyY29uYS9wcm94eXNxbC1lbnRyeXBvaW50LnNoIl0sImFyZ3MiOlsicHJveHlzcWwiLCItZiIsIi1jIiwiL2V0Yy9wcm94eXNxbC9wcm94eXNxbC5jbmYiLCItLXJlbG9hZCJdLCJwb3J0cyI6W3sibmFtZSI6Im15c3FsIiwiY29udGFpbmVyUG9ydCI6MzMwNn0seyJuYW1lIjoicHJveHlhZG0iLCJjb250YWluZXJQb3J0Ijo2MDMyfSx7Im5hbWUiOiJzdGF0cyIsImNvbnRhaW5lclBvcnQiOjYwNzB9XSwiZW52RnJvbSI6W3sic2VjcmV0UmVmIjp7Im5hbWUiOiJjcm9zcy1zaXRlLXJlcGxpY2EtZW52LXZhcnMtcHJveHlzcWwiLCJvcHRpb25hbCI6dHJ1ZX19XSwiZW52IjpbeyJuYW1lIjoiUFhDX1NFUlZJQ0UiLCJ2YWx1ZSI6ImNyb3NzLXNpdGUtcmVwbGljYS1weGMifSx7Im5hbWUiOiJPUEVSQVRPUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLWNyb3NzLXNpdGUtcmVwbGljYSIsImtleSI6Im9wZXJhdG9yIn19fSx7Im5hbWUiOiJQUk9YWV9BRE1JTl9VU0VSIiwidmFsdWUiOiJwcm94eWFkbWluIn0seyJuYW1lIjoiUFJPWFlfQURNSU5fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1jcm9zcy1zaXRlLXJlcGxpY2EiLCJrZXkiOiJwcm94eWFkbWluIn19fSx7Im5hbWUiOiJNT05JVE9SX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtY3Jvc3Mtc2l0ZS1yZXBsaWNhIiwia2V5IjoibW9uaXRvciJ9fX0seyJuYW1lIjoiU0NIRURVTEVSX0NIRUNLVElNRU9VVCIsInZhbHVlIjoiMjAwMCJ9LHsibmFtZSI6IlNDSEVEVUxFUl9XUklURVJBTFNPUkVBREVSIiwidmFsdWUiOiIxIn0seyJuYW1lIjoiU0NIRURVTEVSX1JFVFJZVVAiLCJ2YWx1ZSI6IjEifSx7Im5hbWUiOiJTQ0hFRFVMRVJfUkVUUllET1dOIiwidmFsdWUiOiIzIn0seyJuYW1lIjoiU0NIRURVTEVSX1BJTkdUSU1FT1VUIiwidmFsdWUiOiIxMDAwIn0seyJuYW1lIjoiU0NIRURVTEVSX05PREVDSEVDS0lOVEVSVkFMIiwidmFsdWUiOiIyMDAwIn0seyJuYW1lIjoiU0NIRURVTEVSX01BWENPTk5FQ1RJT05TIiwidmFsdWUiOiIxMDAwIn0seyJuYW1lIjoiUEVSQ09OQV9TQ0hFRFVMRVJfQ0ZHIiwidmFsdWUiOiIvdG1wL3NjaGVkdWxlci1jb25maWcudG9tbCJ9LHsibmFtZSI6IlNDSEVEVUxFUl9FTkFCTEVEIiwidmFsdWUiOiJ0cnVlIn0seyJuYW1lIjoiUFhDX1JFQURfT05MWSIsInZhbHVlIjoiZmFsc2UifV0sInJlc291cmNlcyI6eyJsaW1pdHMiOnsiY3B1IjoiNzAwbSIs"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHJveHlzcWwiLCJhcHAua3ViZXJuZXRlcy5pby9pbnN0YW5jZSI6ImNyb3NzLXNpdGUtcmVwbGljYSIsImFwcC5rdWJlcm5ldGVzLmlvL21hbmFnZWQtYnkiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yIiwiYXBwLmt1YmVybmV0ZXMuaW8vbmFtZSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIiLCJhcHAua3ViZXJuZXRlcy5pby9wYXJ0LW9mIjoicGVyY29uYS14dHJhZGItY2x1c3RlciJ9fSwidGVtcGxhdGUiOnsibWV0YWRhdGEiOnsibGFiZWxzIjp7ImFwcC5rdWJlcm5ldGVzLmlvL2NvbXBvbmVudCI6InByb3h5c3FsIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJjcm9zcy1zaXRlLXJlcGxpY2EiLCJhcHAua3ViZXJuZXRlcy5pby9tYW5hZ2VkLWJ5IjoicGVyY29uYS14dHJhZGItY2x1c3Rlci1vcGVyYXRvciIsImFwcC5rdWJlcm5ldGVzLmlvL25hbWUiOiJwZXJjb25hLXh0cmFkYi1jbHVzdGVyIiwiYXBwLmt1YmVybmV0ZXMuaW8vcGFydC1vZiI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXIifSwiYW5ub3RhdGlvbnMiOnsia3ViZWN0bC5rdWJlcm5ldGVzLmlvL2RlZmF1bHQtY29udGFpbmVyIjoicHJveHlzcWwiLCJwZXJjb25hLmNvbS9jb25maWd1cmF0aW9uLWhhc2giOiJkNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZSIsInBlcmNvbmEuY29tL3NzbC1oYXNoIjoiOWVhMDViYWY3YWM0NjNmOTA4ZDM1YmFlY2JmZWVmMDQiLCJwZXJjb25hLmNvbS9zc2wtaW50ZXJuYWwtaGFzaCI6ImFmMTQzOGMyM2FhOGFjNWVhYWRiODA0MjJiN2I4YjQxIn19LCJzcGVjIjp7InZvbHVtZXMiOlt7Im5hbWUiOiJzc2wtaW50ZXJuYWwiLCJzZWNyZXQiOnsic2VjcmV0TmFtZSI6ImNyb3NzLXNpdGUtcmVwbGljYS1zc2wtaW50ZXJuYWwiLCJvcHRpb25hbCI6dHJ1ZX19LHsibmFtZSI6InNzbCIsInNlY3JldCI6eyJzZWNyZXROYW1lIjoiY3Jvc3Mtc2l0ZS1yZXBsaWNhLXNzbCIsIm9wdGlvbmFsIjpmYWxzZX19LHsibmFtZSI6ImJpbiIsImVtcHR5RGlyIjp7fX1dLCJpbml0Q29udGFpbmVycyI6W3sibmFtZSI6InB4Yy1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjUzOS1kY2RmMWU4YSIsImNvbW1hbmQiOlsiL3B4Yy1pbml0LWVudHJ5cG9pbnQuc2giXSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJjcHUiOiI1MG0iLCJtZW1vcnkiOiI1ME0ifX0sInZvbHVtZU1vdW50cyI6W3sibmFtZSI6ImJpbiIsIm1vdW50UGF0aCI6Ii92YXIvbGliL215c3FsIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifSx7Im5hbWUiOiJwcm94eXNxbC1pbml0IiwiaW1hZ2UiOiJwZXJjb25hbGFiL3BlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3I6UFItMjUzOS1kY2RmMWU4YSIsImNvbW1hbmQiOlsiL3Byb3h5c3FsLWluaXQtZW50cnlwb2ludC5zaCJdLCJyZXNvdXJjZXMiOnsibGltaXRzIjp7ImNwdSI6IjUwbSIsIm1lbW9yeSI6IjUwTSJ9fSwidm9sdW1lTW91bnRzIjpbeyJuYW1lIjoiYmluIiwibW91bnRQYXRoIjoiL29wdC9wZXJjb25hIn1dLCJpbWFnZVB1bGxQb2xpY3kiOiJBbHdheXMifV0sImNvbnRhaW5lcnMiOlt7Im5hbWUiOiJwcm94eXNxbCIsImltYWdlIjoicGVyY29uYWxhYi9wZXJjb25hLXh0cmFkYi1jbHVzdGVyLW9wZXJhdG9yOm1haW4tcHJveHlzcWwiLCJjb21tYW5kIjpbIi9vcHQvcGVyY29uYS9wcm94eXNxbC1lbnRyeXBvaW50LnNoIl0sImFyZ3MiOlsicHJveHlzcWwiLCItZiIsIi1jIiwiL2V0Yy9wcm94eXNxbC9wcm94eXNxbC5jbmYiLCItLXJlbG9hZCJdLCJwb3J0cyI6W3sibmFtZSI6Im15c3FsIiwiY29udGFpbmVyUG9ydCI6MzMwNn0seyJuYW1lIjoicHJveHlhZG0iLCJjb250YWluZXJQb3J0Ijo2MDMyfSx7Im5hbWUiOiJzdGF0cyIsImNvbnRhaW5lclBvcnQiOjYwNzB9XSwiZW52RnJvbSI6W3sic2VjcmV0UmVmIjp7Im5hbWUiOiJjcm9zcy1zaXRlLXJlcGxpY2EtZW52LXZhcnMtcHJveHlzcWwiLCJvcHRpb25hbCI6dHJ1ZX19XSwiZW52IjpbeyJuYW1lIjoiUFhDX1NFUlZJQ0UiLCJ2YWx1ZSI6ImNyb3NzLXNpdGUtcmVwbGljYS1weGMifSx7Im5hbWUiOiJPUEVSQVRPUl9QQVNTV09SRCIsInZhbHVlRnJvbSI6eyJzZWNyZXRLZXlSZWYiOnsibmFtZSI6ImludGVybmFsLWNyb3NzLXNpdGUtcmVwbGljYSIsImtleSI6Im9wZXJhdG9yIn19fSx7Im5hbWUiOiJQUk9YWV9BRE1JTl9VU0VSIiwidmFsdWUiOiJwcm94eWFkbWluIn0seyJuYW1lIjoiUFJPWFlfQURNSU5fUEFTU1dPUkQiLCJ2YWx1ZUZyb20iOnsic2VjcmV0S2V5UmVmIjp7Im5hbWUiOiJpbnRlcm5hbC1jcm9zcy1zaXRlLXJlcGxpY2EiLCJrZXkiOiJwcm94eWFkbWluIn19fSx7Im5hbWUiOiJNT05JVE9SX1BBU1NXT1JEIiwidmFsdWVGcm9tIjp7InNlY3JldEtleVJlZiI6eyJuYW1lIjoiaW50ZXJuYWwtY3Jvc3Mtc2l0ZS1yZXBsaWNhIiwia2V5IjoibW9uaXRvciJ9fX0seyJuYW1lIjoiU0NIRURVTEVSX0NIRUNLVElNRU9VVCIsInZhbHVlIjoiMjAwMCJ9LHsibmFtZSI6IlNDSEVEVUxFUl9XUklURVJBTFNPUkVBREVSIiwidmFsdWUiOiIxIn0seyJuYW1lIjoiU0NIRURVTEVSX1JFVFJZVVAiLCJ2YWx1ZSI6IjEifSx7Im5hbWUiOiJTQ0hFRFVMRVJfUkVUUllET1dOIiwidmFsdWUiOiIzIn0seyJuYW1lIjoiU0NIRURVTEVSX1BJTkdUSU1FT1VUIiwidmFsdWUiOiIxMDAwIn0seyJuYW1lIjoiU0NIRURVTEVSX05PREVDSEVDS0lOVEVSVkFMIiwidmFsdWUiOiIyMDAwIn0seyJuYW1lIjoiU0NIRURVTEVSX01BWENPTk5FQ1RJT05TIiwidmFsdWUiOiIxMDAwIn0seyJuYW1lIjoiUEVSQ09OQV9TQ0hFRFVMRVJfQ0ZHIiwidmFsdWUiOiIvdG1wL3NjaGVkdWxlci1jb25maWcudG9tbCJ9LHsibmFtZSI6IlNDSEVEVUxFUl9FTkFCTEVEIiwidmFsdWUiOiJ0cnVlIn0seyJuYW1lIjoiUFhDX1JFQURfT05MWSIsInZhbHVlIjoiZmFsc2UifV0sInJlc291cmNlcyI6eyJsaW1pdHMiOnsiY3B1IjoiNzAwbSIs"..., +  PeriodSeconds: 0, -  PeriodSeconds: 10, +  PersistentVolumeClaimRetentionPolicy: nil, -  PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", +  Phase: "", -  Phase: "Pending", +  PodManagementPolicy: "", -  PodManagementPolicy: "OrderedReady", +  Protocol: "", -  Protocol: "TCP", +  ReadyReplicas: 0, -  ReadyReplicas: 3, -  Replicas: &0, +  Replicas: 0, +  Replicas: &0, -  Replicas: 3, -  Replicas: &3, +  Replicas: &3, +  ResourceVersion: "", -  ResourceVersion: "1782960267390351011", -  ResourceVersion: "1782960432984591009", -  ResourceVersion: "1782960885865087005", -  ResourceVersion: "1782961057774975022", -  ResourceVersion: "1782961268532079005", -  ResourceVersion: "1782961304705343022", -  ResourceVersion: "1782961464972847005", -  ResourceVersion: "1782961717337327005", -  ResourceVersion: "1782961955389439011", -  ResourceVersion: "1782962003530991009", -  ResourceVersion: "1782962162417455011", +  RestartPolicy: "", -  RestartPolicy: "Always", -  RevisionHistoryLimit: &10, +  RevisionHistoryLimit: nil, +  SchedulerName: "", -  SchedulerName: "default-scheduler", -  Subresource: "status", +  TerminationMessagePath: "", -  TerminationMessagePath: "/dev/termination-log", +  TerminationMessagePolicy: "", -  TerminationMessagePolicy: "File", -  Time: s"2026-07-02 02:43:18 +0000 UTC", -  Time: s"2026-07-02 02:44:27 +0000 UTC", -  Time: s"2026-07-02 02:47:12 +0000 UTC", -  Time: s"2026-07-02 02:53:47 +0000 UTC", -  Time: s"2026-07-02 02:54:45 +0000 UTC", -  Time: s"2026-07-02 02:57:37 +0000 UTC", -  Time: s"2026-07-02 03:01:04 +0000 UTC", -  Time: s"2026-07-02 03:01:08 +0000 UTC", -  Time: s"2026-07-02 03:01:44 +0000 UTC", -  Time: s"2026-07-02 03:03:37 +0000 UTC", -  Time: s"2026-07-02 03:04:24 +0000 UTC", -  Time: s"2026-07-02 03:08:07 +0000 UTC", -  Time: s"2026-07-02 03:08:37 +0000 UTC", -  Time: s"2026-07-02 03:12:31 +0000 UTC", -  Time: s"2026-07-02 03:12:35 +0000 UTC", -  Time: s"2026-07-02 03:13:23 +0000 UTC", -  Time: s"2026-07-02 03:15:16 +0000 UTC", -  Time: s"2026-07-02 03:16:02 +0000 UTC", -  TopologySpreadConstraints: nil, +  TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, +  UID: "", -  UID: "08611693-379e-41c8-ac75-34a256e46b44", -  UID: "09397b29-ec37-43b7-96d0-26b9170e1e48", -  UID: "d8ef6591-7d04-45de-b25f-edf165f716e0", -  UID: "e57351ba-b7a9-4b19-a3e3-a1c4d02dbc1e", +  UpdatedReplicas: 0, -  UpdatedReplicas: 3, +  UpdateRevision: "", -  UpdateRevision: "cross-site-replica-proxysql-577966cf8b", -  UpdateRevision: "cross-site-replica-proxysql-67874c6964", -  UpdateRevision: "cross-site-replica-pxc-8674f56b59", -  UpdateRevision: "cross-site-source-proxysql-664db96fdb", -  UpdateRevision: "cross-site-source-pxc-6988c69dbf", -  Value: "false", +  Value: "false", -  Value: "true", +  Value: "true", -  VolumeMode: &"Filesystem", +  VolumeMode: nil,   }    },    },    {    },    },    {    },    },    {    },    ... // 12 identical elements    ... // 16 identical fields    ... // 16 identical fields    ... // 22 identical fields    ... // 2 identical fields    ... // 3 identical fields    ... // 3 identical fields    ... // 3 identical fields    ... // 4 identical fields    ... // 5 identical fields    ... // 5 identical fields    ... // 6 identical fields    ... // 6 identical fields    ... // 7 identical fields    ... // 8 identical fields    ... // 9 identical fields    AccessModes: nil,    ActiveDeadlineSeconds: nil,    Affinity: nil,    Affinity: &{PodAntiAffinity: &{RequiredDuringSchedulingIgnoredDuringExecution: {{LabelSelector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "cross-site-source", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}}, TopologyKey: "kubernetes.io/hostname"}}}},    Args: {"mysqld"},    Args: {"proxysql", "-f", "-c", "/etc/proxysql/proxysql.cnf", ...},    AutomountServiceAccountToken: nil,    AvailableReplicas: 0,    AWSElasticBlockStore: nil,    AzureFile: nil,    Capacity: nil,    Conditions: nil,    ConfigMap: &v1.ConfigMapVolumeSource{    ContainerPort: 3306,    ContainerPort: 33060,    ContainerPort: 33062,    ContainerPort: 4444,    ContainerPort: 4567,    ContainerPort: 4568,    ContainerPort: 6032,    ContainerPort: 6070,    Containers: []v1.Container{    CurrentReplicas: 0,    DataSource: nil,    DataSourceRef: nil,    DeletionGracePeriodSeconds: nil,    DeletionTimestamp: nil,    EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "cross-site-replica-env-vars-proxysql"}, Optional: &true}}},    EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "cross-site-replica-env-vars-pxc"}, Optional: &true}}},    EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "cross-site-source-env-vars-proxysql"}, Optional: &true}}},    EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "cross-site-source-env-vars-pxc"}, Optional: &true}}},    Env: {{Name: "PXC_SERVICE", Value: "cross-site-replica-pxc"}, {Name: "OPERATOR_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-cross-site-replica"}, Key: "operator"}}}, {Name: "PROXY_ADMIN_USER", Value: "proxyadmin"}, {Name: "PROXY_ADMIN_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-cross-site-replica"}, Key: "proxyadmin"}}}, ...},    Env: {{Name: "PXC_SERVICE", Value: "cross-site-replica-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-cross-site-replica"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-cross-site-replica"}, Key: "xtrabackup"}}}, ...},    Env: {{Name: "PXC_SERVICE", Value: "cross-site-source-pxc"}, {Name: "OPERATOR_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-cross-site-source"}, Key: "operator"}}}, {Name: "PROXY_ADMIN_USER", Value: "proxyadmin"}, {Name: "PROXY_ADMIN_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-cross-site-source"}, Key: "proxyadmin"}}}, ...},    Env: {{Name: "PXC_SERVICE", Value: "cross-site-source-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-cross-site-source"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-cross-site-source"}, Key: "xtrabackup"}}}, ...},    Env: []v1.EnvVar{    EphemeralContainers: nil,    FailureThreshold: 3,    FC: nil,    Finalizers: nil,    GitRepo: nil,    HostAliases: nil,    HostIP: "",    HostPort: 0,    ImagePullPolicy: "Always",    InitContainers: []v1.Container{    InitialDelaySeconds: 300,    ISCSI: nil,    Items: nil,    Items: nil,    Labels: nil,    Lifecycle: nil,    LivenessProbe: &v1.Probe{    LocalObjectReference: {Name: "auto-cross-site-replica-pxc"},    LocalObjectReference: {Name: "auto-cross-site-source-pxc"},    LocalObjectReference: {Name: "cross-site-replica-pxc"},    LocalObjectReference: {Name: "cross-site-source-pxc"},    MinReadySeconds: 0,    Name: "auto-config",    {Name: "bin", VolumeSource: {EmptyDir: &{}}},    Name: "config",    Name: "ist",    Name: "mysql",    Name: "mysql-admin",    Name: "mysql-init-file",    Name: "mysql-users-secret-file",    Name: "mysqlx",    {Name: "PERCONA_SCHEDULER_CFG", Value: "/tmp/scheduler-config.toml"},    Name: "proxyadm",    Name: "PXC_READ_ONLY",    {Name: "SCHEDULER_ENABLED", Value: "true"},    Namespace: "cross-site-proxysql-28035",    Namespace: "cross-site-proxysql-replica-17697",    Name: "ssl",    Name: "ssl-internal",    Name: "sst",    Name: "stats",    {Name: "tmp", VolumeSource: {EmptyDir: &{}}},    Name: "vault-keyring-secret",    Name: "write-set",    NFS: nil,    NodeName: "",    NodeSelector: nil,    ObjectMeta: {Labels: {"app.kubernetes.io/component": "proxysql", "app.kubernetes.io/instance": "cross-site-replica", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "proxysql", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "9ea05baf7ac463f908d35baecbfeef04", "percona.com/ssl-internal-hash": "af1438c23aa8ac5eaadb80422b7b8b41"}},    ObjectMeta: {Labels: {"app.kubernetes.io/component": "proxysql", "app.kubernetes.io/instance": "cross-site-source", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "proxysql", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "e7a5c1b19a6b63f0bc6e425e9b5ec21c", "percona.com/ssl-internal-hash": "af1438c23aa8ac5eaadb80422b7b8b41"}},    ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "cross-site-replica", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "9ea05baf7ac463f908d35baecbfeef04", "percona.com/ssl-internal-hash": "af1438c23aa8ac5eaadb80422b7b8b41"}},    ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "cross-site-source", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "e7a5c1b19a6b63f0bc6e425e9b5ec21c", "percona.com/ssl-internal-hash": "af1438c23aa8ac5eaadb80422b7b8b41"}},    ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "cross-site-replica", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "cross-site-source", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    ObjectMeta: {Name: "proxydata", Labels: {"app.kubernetes.io/component": "proxysql", "app.kubernetes.io/instance": "cross-site-replica", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    ObjectMeta: {Name: "proxydata", Labels: {"app.kubernetes.io/component": "proxysql", "app.kubernetes.io/instance": "cross-site-source", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    ObjectMeta: v1.ObjectMeta{    Optional: &false,    Optional: &true,    Optional: &true,    Ordinals: nil,    OS: nil,    Overhead: nil,    OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "cross-site-replica", UID: "b64fed3e-f05c-4a45-a7c5-7b33e75d337f", ...}},    OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "cross-site-source", UID: "4520e5bb-9c43-44ad-8ca7-e1b838ffddb3", ...}},    Ports: nil,    Ports: []v1.ContainerPort{    PreemptionPolicy: nil,    ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}},    Quobyte: nil,    ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...},    ReadyReplicas: 0,    Replicas: 0,    Replicas: &3,    ResizePolicy: nil,    Resources: {},    Resources: {Limits: {s"cpu": {i: {...}, s: "700m", Format: "DecimalSI"}, s"memory": {i: {...}, s: "1G", Format: "DecimalSI"}}, Requests: {s"cpu": {i: {...}, s: "100m", Format: "DecimalSI"}, s"memory": {i: {...}, s: "100M", Format: "DecimalSI"}}},    SecretName: "cross-site-replica-mysql-init",    SecretName: "cross-site-replica-ssl",    SecretName: "cross-site-replica-ssl-internal",    SecretName: "cross-site-source-mysql-init",    SecretName: "cross-site-source-ssl",    SecretName: "cross-site-source-ssl-internal",    SecretName: "internal-cross-site-replica",    SecretName: "internal-cross-site-source",    SecretName: "some-name-vault",    Secret: &v1.SecretVolumeSource{    SecurityContext: nil,    Selector: &{MatchLabels: {"app.kubernetes.io/component": "proxysql", "app.kubernetes.io/instance": "cross-site-replica", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    Selector: &{MatchLabels: {"app.kubernetes.io/component": "proxysql", "app.kubernetes.io/instance": "cross-site-source", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "cross-site-replica", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "cross-site-source", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    SelfLink: "",    ServiceAccountName: "default",    ServiceName: "cross-site-replica-proxysql-unready",    ServiceName: "cross-site-replica-pxc",    ServiceName: "cross-site-source-proxysql-unready",    ServiceName: "cross-site-source-pxc",    SetHostnameAsFQDN: nil,    Spec: v1.PersistentVolumeClaimSpec{    Spec: v1.PodSpec{    Spec: v1.StatefulSetSpec{    StartupProbe: nil,    Status: v1.PersistentVolumeClaimStatus{    Status: v1.StatefulSetStatus{    StorageClassName: nil,    Subdomain: "",    SuccessThreshold: 1,    Template: v1.PodTemplateSpec{    TerminationGracePeriodSeconds: &30,    TerminationGracePeriodSeconds: &600,    TerminationGracePeriodSeconds: nil,    TimeoutSeconds: 5,    Tolerations: nil,    TypeMeta: {},    TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"},    UpdatedReplicas: 0,    UpdateStrategy: {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}},   &v1.StatefulSet{    ValueFrom: nil,    VolumeAttributesClassName: nil,    VolumeClaimTemplates: []v1.PersistentVolumeClaim{    VolumeDevices: nil,    VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...},    VolumeName: "",    VolumeSource: v1.VolumeSource{    Volumes: []v1.Volume{    VsphereVolume: nil,    WorkingDir: "", + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names='cross-site-proxysql-28035/backup-minio-source cross-site-proxysql-replica-17697/backup-minio-replica' + [[ -n cross-site-proxysql-28035/backup-minio-source cross-site-proxysql-replica-17697/backup-minio-replica ]] + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found perconaxtradbclusterbackup.pxc.percona.com "backup-minio-source" deleted from cross-site-proxysql-28035 namespace perconaxtradbclusterbackup.pxc.percona.com "backup-minio-replica" deleted from cross-site-proxysql-replica-17697 namespace ++ date +%s + local deadline=1782962946 ++ kubectl_bin get pxc-backup --all-namespaces -o name --ignore-not-found + [[ -z '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names='cross-site-proxysql-28035/cross-site-source cross-site-proxysql-replica-17697/cross-site-replica' + [[ -n cross-site-proxysql-28035/cross-site-source cross-site-proxysql-replica-17697/cross-site-replica ]] + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + kubectl_bin patch pxc -n cross-site-proxysql-28035 cross-site-source --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + read -r pns name + kubectl_bin patch pxc -n cross-site-proxysql-replica-17697 cross-site-replica --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --ignore-not-found perconaxtradbcluster.pxc.percona.com "cross-site-source" deleted from cross-site-proxysql-28035 namespace perconaxtradbcluster.pxc.percona.com "cross-site-replica" deleted from cross-site-proxysql-replica-17697 namespace ++ date +%s + local deadline=1782962968 ++ kubectl_bin get pxc --all-namespaces -o name --ignore-not-found + [[ -z '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found perconaxtradbclusterrestore.pxc.percona.com "backup-minio" deleted from cross-site-proxysql-28035 namespace perconaxtradbclusterrestore.pxc.percona.com "backup-minio" deleted from cross-site-proxysql-replica-17697 namespace + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.Jj5LQOhl47 ++ mktemp + local LAST_ERR=/tmp/tmp.5mMlt6Xwg3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Jj5LQOhl47 validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.5mMlt6Xwg3 + rm /tmp/tmp.Jj5LQOhl47 /tmp/tmp.5mMlt6Xwg3 + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ date +%s + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-proxysql-28035 + local deadline=1782962819 ++ mktemp + local LAST_OUT=/tmp/tmp.4sl7KYrQh0 ++ mktemp ++ kubectl_bin get namespace cross-site-proxysql-28035 --ignore-not-found -o name ++ mktemp + local LAST_OUT=/tmp/tmp.ounLCTKTi9 + local LAST_ERR=/tmp/tmp.aigRleSQuJ + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.s0KUebMCgx + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace cross-site-proxysql-28035 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + [[ -z namespace/cross-site-proxysql-28035 ]] ++ date +%s + [[ 1782962701 -ge 1782962819 ]] + sleep 5 ++ kubectl_bin get namespace cross-site-proxysql-28035 --ignore-not-found -o name + [[ -z namespace/cross-site-proxysql-28035 ]] ++ date +%s + [[ 1782962707 -ge 1782962819 ]] + sleep 5 ++ kubectl_bin get namespace cross-site-proxysql-28035 --ignore-not-found -o name + [[ -z namespace/cross-site-proxysql-28035 ]] ++ date +%s + [[ 1782962715 -ge 1782962819 ]] + sleep 5 ++ kubectl_bin get namespace cross-site-proxysql-28035 --ignore-not-found -o name + [[ -z namespace/cross-site-proxysql-28035 ]] ++ date +%s + [[ 1782962721 -ge 1782962819 ]] + sleep 5 ++ kubectl_bin get namespace cross-site-proxysql-28035 --ignore-not-found -o name + [[ -z namespace/cross-site-proxysql-28035 ]] ++ date +%s + [[ 1782962729 -ge 1782962819 ]] + sleep 5 ++ kubectl_bin get namespace cross-site-proxysql-28035 --ignore-not-found -o name + [[ -z namespace/cross-site-proxysql-28035 ]] ++ date +%s + [[ 1782962736 -ge 1782962819 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ounLCTKTi9 namespace "cross-site-proxysql-28035" force deleted + cat /tmp/tmp.s0KUebMCgx Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.ounLCTKTi9 /tmp/tmp.s0KUebMCgx + return 0 ++ kubectl_bin get namespace cross-site-proxysql-28035 --ignore-not-found -o name + [[ -z '' ]] ++ date +%s + deadline=1782962864 ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' ++ awk '$2 != ""' + [[ -z '' ]] + rm -rf /tmp/tmp.xyHcWb7plX + destroy cross-site-proxysql-replica-17697 true + local namespace=cross-site-proxysql-replica-17697 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'the object has been modified' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + grep -v 'get backup status: Job.batch' + grep -v level=info + sort -u + tee /tmp/tmp.xyHcWb7plX/operator.log tee: /tmp/tmp.xyHcWb7plX/operator.log: No such file or directory +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator No resources found in pxc-operator namespace. ++ local check_label=0 ++ [[ 0 -eq 0 ]] ++ label_prefix= ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.fISTyQQVpS +++ mktemp ++ local LAST_ERR=/tmp/tmp.r64KyrPYYI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fISTyQQVpS ++ cat /tmp/tmp.r64KyrPYYI ++ rm /tmp/tmp.fISTyQQVpS /tmp/tmp.r64KyrPYYI ++ return 0 + kubectl_bin logs -n pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.9QWJP9aaq7 ++ mktemp + local LAST_ERR=/tmp/tmp.ymWMU59KkS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.9QWJP9aaq7 + cat /tmp/tmp.ymWMU59KkS error: expected 'logs [-f] [-p] (POD | TYPE/NAME) [-c CONTAINER]'. POD or TYPE/NAME is a required argument for the logs command See 'kubectl logs -h' for help and examples + rm /tmp/tmp.9QWJP9aaq7 /tmp/tmp.ymWMU59KkS + return 1 + : + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names= + [[ -n '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names= + [[ -n '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.07SNavZCUj ++ mktemp + local LAST_ERR=/tmp/tmp.oF6RCtiOqa + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.07SNavZCUj + cat /tmp/tmp.oF6RCtiOqa + rm /tmp/tmp.07SNavZCUj /tmp/tmp.oF6RCtiOqa + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ date +%s + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-proxysql-replica-17697 + local deadline=1782962903 ++ mktemp ++ kubectl_bin get namespace cross-site-proxysql-replica-17697 --ignore-not-found -o name + local LAST_OUT=/tmp/tmp.jIOtAPvR39 ++ mktemp + local LAST_OUT=/tmp/tmp.SrLifm7Sia ++ mktemp + local LAST_ERR=/tmp/tmp.E3k3gmdQAN + local exit_status=0 ++ mktemp ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + local LAST_ERR=/tmp/tmp.gaYWxWeMrJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace cross-site-proxysql-replica-17697 + [[ -z namespace/cross-site-proxysql-replica-17697 ]] ++ date +%s + [[ 1782962787 -ge 1782962903 ]] + sleep 5 ++ kubectl_bin get namespace cross-site-proxysql-replica-17697 --ignore-not-found -o name + [[ -z namespace/cross-site-proxysql-replica-17697 ]] ++ date +%s + [[ 1782962795 -ge 1782962903 ]] + sleep 5 ++ kubectl_bin get namespace cross-site-proxysql-replica-17697 --ignore-not-found -o name + [[ -z namespace/cross-site-proxysql-replica-17697 ]] ++ date +%s + [[ 1782962802 -ge 1782962903 ]] + sleep 5 ++ kubectl_bin get namespace cross-site-proxysql-replica-17697 --ignore-not-found -o name + [[ -z namespace/cross-site-proxysql-replica-17697 ]] ++ date +%s + [[ 1782962809 -ge 1782962903 ]] + sleep 5 ++ kubectl_bin get namespace cross-site-proxysql-replica-17697 --ignore-not-found -o name + [[ -z namespace/cross-site-proxysql-replica-17697 ]] ++ date +%s + [[ 1782962816 -ge 1782962903 ]] + sleep 5 ++ kubectl_bin get namespace cross-site-proxysql-replica-17697 --ignore-not-found -o name + [[ -z namespace/cross-site-proxysql-replica-17697 ]] ++ date +%s + [[ 1782962822 -ge 1782962903 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jIOtAPvR39 namespace "cross-site-proxysql-replica-17697" force deleted + cat /tmp/tmp.gaYWxWeMrJ Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.jIOtAPvR39 /tmp/tmp.gaYWxWeMrJ + return 0 ++ kubectl_bin get namespace cross-site-proxysql-replica-17697 --ignore-not-found -o name + [[ -z '' ]] ++ date +%s + deadline=1782962949 ++ awk '$2 != ""' ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' + [[ -z '' ]] + rm -rf /tmp/tmp.xyHcWb7plX + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------