Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/logs/cross-site-8-0.log WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + main + local source_cluster=cross-site-source + local replica_cluster=cross-site-replica + desc 'Create source cluster' + set +o xtrace ----------------------------------------------------------------------------------- Create source cluster ----------------------------------------------------------------------------------- + create_infra cross-site-21497 + local ns=cross-site-21497 + '[' -n pxc-operator ']' + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get pxc --all-namespaces -o wide + kubectl patch pxc -n cross-site-10488 cross-site-source --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.lObgNHYkE0 ++ mktemp + local LAST_ERR=/tmp/tmp.q8GzVJaZuG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lObgNHYkE0 perconaxtradbcluster.pxc.percona.com "cross-site-source" deleted + cat /tmp/tmp.q8GzVJaZuG + rm /tmp/tmp.lObgNHYkE0 /tmp/tmp.q8GzVJaZuG + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.4s2ASTB6xU ++ mktemp + local LAST_ERR=/tmp/tmp.lYJYZ3MlQt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4s2ASTB6xU No resources found + cat /tmp/tmp.lYJYZ3MlQt + rm /tmp/tmp.4s2ASTB6xU /tmp/tmp.lYJYZ3MlQt + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.GiDf91hpCn ++ mktemp + local LAST_ERR=/tmp/tmp.pCrLhPHq5R + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GiDf91hpCn No resources found + cat /tmp/tmp.pCrLhPHq5R + rm /tmp/tmp.GiDf91hpCn /tmp/tmp.pCrLhPHq5R + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.ytgwFyCuNp + local LAST_OUT=/tmp/tmp.2zhNftxgFM ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.nH6kHFybdq + local LAST_ERR=/tmp/tmp.XWYpqgdq5p + local exit_status=0 + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2zhNftxgFM + cat /tmp/tmp.XWYpqgdq5p + rm /tmp/tmp.2zhNftxgFM /tmp/tmp.XWYpqgdq5p + return 0 namespace "cross-site-10488" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ytgwFyCuNp namespace "pxc-operator" deleted + cat /tmp/tmp.nH6kHFybdq + rm /tmp/tmp.ytgwFyCuNp /tmp/tmp.nH6kHFybdq + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'namespace/pxc-operator - ' namespace/pxc-operator - + set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.Cn8dP9dVuo ++ mktemp + local LAST_ERR=/tmp/tmp.42Y8RBGfFy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Cn8dP9dVuo namespace/pxc-operator created + cat /tmp/tmp.42Y8RBGfFy + rm /tmp/tmp.Cn8dP9dVuo /tmp/tmp.42Y8RBGfFy + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.vd4EI6DN4e +++ mktemp ++ local LAST_ERR=/tmp/tmp.sQPxpcjf3g ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vd4EI6DN4e ++ cat /tmp/tmp.sQPxpcjf3g ++ rm /tmp/tmp.vd4EI6DN4e /tmp/tmp.sQPxpcjf3g ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.7xf3BADwHL ++ mktemp + local LAST_ERR=/tmp/tmp.AoB1D7EOB5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7xf3BADwHL Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3" modified. + cat /tmp/tmp.AoB1D7EOB5 + rm /tmp/tmp.7xf3BADwHL /tmp/tmp.AoB1D7EOB5 + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.bdyjby1l00 ++ mktemp + local LAST_ERR=/tmp/tmp.TNOelwXxjI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bdyjby1l00 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.TNOelwXxjI + rm /tmp/tmp.bdyjby1l00 /tmp/tmp.TNOelwXxjI + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/cw-rbac.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.kuJRpWqm5A ++ mktemp + local LAST_ERR=/tmp/tmp.AsWqpJLVYQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kuJRpWqm5A clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.AsWqpJLVYQ + rm /tmp/tmp.kuJRpWqm5A /tmp/tmp.AsWqpJLVYQ + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - ++ mktemp + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - + local LAST_OUT=/tmp/tmp.gHcdolJjTQ ++ mktemp + local LAST_ERR=/tmp/tmp.PjlbHPDGPW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gHcdolJjTQ deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.PjlbHPDGPW + rm /tmp/tmp.gHcdolJjTQ /tmp/tmp.PjlbHPDGPW + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.2Ub5kxEe0h ++ mktemp + local LAST_ERR=/tmp/tmp.x5oFZPtcQh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2Ub5kxEe0h pod/percona-xtradb-cluster-operator-675fc7d9f5-jjk9g condition met + cat /tmp/tmp.x5oFZPtcQh + rm /tmp/tmp.2Ub5kxEe0h /tmp/tmp.x5oFZPtcQh + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.DAKg3HuKG7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.pDUdpbQ0jQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DAKg3HuKG7 ++ cat /tmp/tmp.pDUdpbQ0jQ ++ rm /tmp/tmp.DAKg3HuKG7 /tmp/tmp.pDUdpbQ0jQ ++ return 0 + wait_pod percona-xtradb-cluster-operator-675fc7d9f5-jjk9g 480 pxc-operator + local pod=percona-xtradb-cluster-operator-675fc7d9f5-jjk9g + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-675fc7d9f5-jjk9g ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-675fc7d9f5-jjk9g condition met percona-xtradb-cluster-operator-675fc7d9f5-jjk9g.Ok + sleep 3 + create_namespace cross-site-21497 + local namespace=cross-site-21497 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl api-resources ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrole ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-21497' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-21497 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-21497 + kubectl_bin get ns + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.4kREm39V1M ++ mktemp + xargs kubectl delete ns + awk '{print$1}' + local LAST_ERR=/tmp/tmp.6ZTzP341DT + local exit_status=0 ++ mktemp + local LAST_OUT=/tmp/tmp.P1zSv8EoQz ++ seq 0 2 ++ mktemp + local LAST_ERR=/tmp/tmp.6DT0gDG8BV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-21497 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.P1zSv8EoQz + cat /tmp/tmp.6DT0gDG8BV + rm /tmp/tmp.P1zSv8EoQz /tmp/tmp.6DT0gDG8BV + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-21497 Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-21497 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.4kREm39V1M + cat /tmp/tmp.6ZTzP341DT Error from server (NotFound): namespaces "cross-site-21497" not found + rm /tmp/tmp.4kREm39V1M /tmp/tmp.6ZTzP341DT + return 1 + : + wait_for_delete namespace/cross-site-21497 + local res=namespace/cross-site-21497 + echo -n 'namespace/cross-site-21497 - ' namespace/cross-site-21497 - + set +o xtrace Error from server (NotFound): namespaces "cross-site-21497" not found + desc 'create namespace cross-site-21497' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-21497 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-21497 ++ mktemp + local LAST_OUT=/tmp/tmp.jIzKG5C1kN ++ mktemp + local LAST_ERR=/tmp/tmp.JsZbknswo8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cross-site-21497 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jIzKG5C1kN namespace/cross-site-21497 created + cat /tmp/tmp.JsZbknswo8 + rm /tmp/tmp.jIzKG5C1kN /tmp/tmp.JsZbknswo8 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.hTVX6QhJm1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.N7VRtuufvA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hTVX6QhJm1 ++ cat /tmp/tmp.N7VRtuufvA ++ rm /tmp/tmp.hTVX6QhJm1 /tmp/tmp.N7VRtuufvA ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3 --namespace=cross-site-21497 ++ mktemp + local LAST_OUT=/tmp/tmp.jKS5QR4zsI ++ mktemp + local LAST_ERR=/tmp/tmp.OsIvRTYVXK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3 --namespace=cross-site-21497 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jKS5QR4zsI Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3" modified. + cat /tmp/tmp.OsIvRTYVXK + rm /tmp/tmp.jKS5QR4zsI /tmp/tmp.OsIvRTYVXK + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.TuDFZPV32m ++ mktemp + local LAST_ERR=/tmp/tmp.Z6Qmu9kFiu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TuDFZPV32m secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.Z6Qmu9kFiu + rm /tmp/tmp.TuDFZPV32m /tmp/tmp.Z6Qmu9kFiu + return 0 + start_minio + deploy_helm cross-site-21497 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "chaos-mesh" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Wed Jul 10 13:20:20 2024 NAMESPACE: cross-site-21497 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.cross-site-21497.svc.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace cross-site-21497 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace cross-site-21497 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace cross-site-21497 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace cross-site-21497 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NXKK7Aubd2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.PDSHTS8vLB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NXKK7Aubd2 ++ cat /tmp/tmp.PDSHTS8vLB ++ rm /tmp/tmp.NXKK7Aubd2 /tmp/tmp.PDSHTS8vLB ++ return 0 + MINIO_POD=minio-service-76ffcfd45-gd8x2 + wait_pod minio-service-76ffcfd45-gd8x2 + local pod=minio-service-76ffcfd45-gd8x2 + local max_retry=480 + local ns= ++ echo minio-service-76ffcfd45-gd8x2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minio-service-76ffcfd45-gd8x2 condition met minio-service-76ffcfd45-gd8x2.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.gIXnanyBi5 ++ mktemp + local LAST_ERR=/tmp/tmp.xJSG7GUxam + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gIXnanyBi5 make_bucket: operator-testing pod "aws-cli" deleted + cat /tmp/tmp.xJSG7GUxam If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: error attaching to container: container is in CONTAINER_EXITED state + rm /tmp/tmp.gIXnanyBi5 /tmp/tmp.xJSG7GUxam + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.grAhbZKgh4 ++ mktemp + local LAST_ERR=/tmp/tmp.kRBd7xMxbv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.grAhbZKgh4 secret/minio-secret unchanged secret/aws-s3-secret unchanged secret/gcp-cs-secret unchanged secret/azure-secret unchanged + cat /tmp/tmp.kRBd7xMxbv + rm /tmp/tmp.grAhbZKgh4 /tmp/tmp.kRBd7xMxbv + return 0 + spinup_pxc cross-site-source /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/cross-site-source.yml + local cluster=cross-site-source + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/cross-site-source.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.dS8L4kitE3 ++ mktemp + local LAST_ERR=/tmp/tmp.Rw8yl34b6E + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dS8L4kitE3 secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.Rw8yl34b6E + rm /tmp/tmp.dS8L4kitE3 /tmp/tmp.Rw8yl34b6E + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-21497~ + local LAST_OUT=/tmp/tmp.OIDf7bGIVA ++ mktemp + local LAST_ERR=/tmp/tmp.528YLpb5yI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OIDf7bGIVA deployment.apps/pxc-client created + cat /tmp/tmp.528YLpb5yI + rm /tmp/tmp.OIDf7bGIVA /tmp/tmp.528YLpb5yI + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/cross-site-source.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/cross-site-source.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/cross-site-source.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-21497~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.Vk0AcnvI9m ++ mktemp + local LAST_ERR=/tmp/tmp.ANeDKdtbJM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Vk0AcnvI9m perconaxtradbcluster.pxc.percona.com/cross-site-source created + cat /tmp/tmp.ANeDKdtbJM + rm /tmp/tmp.Vk0AcnvI9m /tmp/tmp.ANeDKdtbJM + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy cross-site-source ++ local target_cluster=cross-site-source +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.YNlQQLlmEv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zv1mC0Jgzu +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.YNlQQLlmEv +++ cat /tmp/tmp.zv1mC0Jgzu +++ rm /tmp/tmp.YNlQQLlmEv /tmp/tmp.zv1mC0Jgzu +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo cross-site-source-haproxy ++ return + local proxy=cross-site-source-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-21497 ++ mktemp + local LAST_OUT=/tmp/tmp.8NpuxB8rDe ++ mktemp + local LAST_ERR=/tmp/tmp.QGzHmxrchQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-21497 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-21497 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-21497 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.8NpuxB8rDe + cat /tmp/tmp.QGzHmxrchQ error: no matching resources found + rm /tmp/tmp.8NpuxB8rDe /tmp/tmp.QGzHmxrchQ + return 1 + true + wait_for_running cross-site-source-haproxy 1 + local name=cross-site-source-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-haproxy-0 480 + local pod=cross-site-source-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo cross-site-source-haproxy-0 + local container= + set +o xtrace Error from server (NotFound): pods "cross-site-source-haproxy-0" not found cross-site-source-haproxy-0............................................Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running cross-site-source-pxc 3 + local name=cross-site-source-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-0 480 + local pod=cross-site-source-pxc-0 + local max_retry=480 + local ns= ++ echo cross-site-source-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-source-pxc-0 condition met cross-site-source-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-1 480 + local pod=cross-site-source-pxc-1 + local max_retry=480 + local ns= ++ echo cross-site-source-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-source-pxc-1 condition met cross-site-source-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-2 480 + local pod=cross-site-source-pxc-2 + local max_retry=480 + local ns= ++ echo cross-site-source-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-source-pxc-2 condition met cross-site-source-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h cross-site-source-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h cross-site-source-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ORHHj1aQPz +++ mktemp ++ local LAST_ERR=/tmp/tmp.7VI2EnEgk7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ORHHj1aQPz ++ cat /tmp/tmp.7VI2EnEgk7 ++ rm /tmp/tmp.ORHHj1aQPz /tmp/tmp.7VI2EnEgk7 ++ return 0 + client_pod=pxc-client-6644d8898f-4dc42 + wait_pod pxc-client-6644d8898f-4dc42 + local pod=pxc-client-6644d8898f-4dc42 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-4dc42 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h cross-site-source-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h cross-site-source-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Mgj1lbh7cS +++ mktemp ++ local LAST_ERR=/tmp/tmp.1qo3SZaVrZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Mgj1lbh7cS ++ cat /tmp/tmp.1qo3SZaVrZ ++ rm /tmp/tmp.Mgj1lbh7cS /tmp/tmp.1qo3SZaVrZ ++ return 0 + client_pod=pxc-client-6644d8898f-4dc42 + wait_pod pxc-client-6644d8898f-4dc42 + local pod=pxc-client-6644d8898f-4dc42 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-4dc42 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HvJpOdgUZJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.HQ2cx0L14S ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HvJpOdgUZJ ++ cat /tmp/tmp.HQ2cx0L14S ++ rm /tmp/tmp.HvJpOdgUZJ /tmp/tmp.HQ2cx0L14S ++ return 0 + client_pod=pxc-client-6644d8898f-4dc42 + wait_pod pxc-client-6644d8898f-4dc42 + local pod=pxc-client-6644d8898f-4dc42 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-4dc42 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Oz5Ph9gkPr/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.Oz5Ph9gkPr/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Y2jgBCjslj +++ mktemp ++ local LAST_ERR=/tmp/tmp.jfDq2uSOJO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Y2jgBCjslj ++ cat /tmp/tmp.jfDq2uSOJO ++ rm /tmp/tmp.Y2jgBCjslj /tmp/tmp.jfDq2uSOJO ++ return 0 + client_pod=pxc-client-6644d8898f-4dc42 + wait_pod pxc-client-6644d8898f-4dc42 + local pod=pxc-client-6644d8898f-4dc42 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6644d8898f-4dc42 ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Oz5Ph9gkPr/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.Oz5Ph9gkPr/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6mYza27kkH +++ mktemp ++ local LAST_ERR=/tmp/tmp.5Aqc9lwnb7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6mYza27kkH ++ cat /tmp/tmp.5Aqc9lwnb7 ++ rm /tmp/tmp.6mYza27kkH /tmp/tmp.5Aqc9lwnb7 ++ return 0 + client_pod=pxc-client-6644d8898f-4dc42 + wait_pod pxc-client-6644d8898f-4dc42 + local pod=pxc-client-6644d8898f-4dc42 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-4dc42 + local container= + set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Oz5Ph9gkPr/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.Oz5Ph9gkPr/select-1.sql ++ is_keyring_plugin_in_use cross-site-source ++ local cluster=cross-site-source ++ kubectl_bin exec -it cross-site-source-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gMOgNfDntf +++ mktemp ++ local LAST_ERR=/tmp/tmp.D20CZVGNDO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it cross-site-source-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gMOgNfDntf ++ cat /tmp/tmp.D20CZVGNDO Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.gMOgNfDntf /tmp/tmp.D20CZVGNDO ++ return 0 + '[' '' ']' + sleep 60 + desc 'get main cluster services endpoints' + set +o xtrace ----------------------------------------------------------------------------------- get main cluster services endpoints ----------------------------------------------------------------------------------- ++ get_service_ip cross-site-source-pxc-0 ++ local service=cross-site-source-pxc-0 ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pmdW8OY3ZX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qdwPbmCxNS +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.pmdW8OY3ZX +++ cat /tmp/tmp.qdwPbmCxNS +++ rm /tmp/tmp.pmdW8OY3ZX /tmp/tmp.qdwPbmCxNS +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LMmOfI1F4U +++ mktemp ++ local LAST_ERR=/tmp/tmp.fi0TLOb0f6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LMmOfI1F4U ++ cat /tmp/tmp.fi0TLOb0f6 ++ rm /tmp/tmp.LMmOfI1F4U /tmp/tmp.fi0TLOb0f6 ++ return 0 ++ return + source_endpoint0=10.119.165.152 ++ get_service_ip cross-site-source-pxc-1 ++ local service=cross-site-source-pxc-1 ++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0w9BGrhIkt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uN2ZmzjvVL +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.0w9BGrhIkt +++ cat /tmp/tmp.uN2ZmzjvVL +++ rm /tmp/tmp.0w9BGrhIkt /tmp/tmp.uN2ZmzjvVL +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fMK6FJKuUG +++ mktemp ++ local LAST_ERR=/tmp/tmp.cH9D99AngN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fMK6FJKuUG ++ cat /tmp/tmp.cH9D99AngN ++ rm /tmp/tmp.fMK6FJKuUG /tmp/tmp.cH9D99AngN ++ return 0 ++ return + source_endpoint1=10.119.168.71 ++ get_service_ip cross-site-source-pxc-2 ++ local service=cross-site-source-pxc-2 ++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.aJTzlLdqUx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NrlrQqMayO +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.aJTzlLdqUx +++ cat /tmp/tmp.NrlrQqMayO +++ rm /tmp/tmp.aJTzlLdqUx /tmp/tmp.NrlrQqMayO +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.N2PXnq8irZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.5iH7fuB2xG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.N2PXnq8irZ ++ cat /tmp/tmp.5iH7fuB2xG ++ rm /tmp/tmp.N2PXnq8irZ /tmp/tmp.5iH7fuB2xG ++ return 0 ++ return + source_endpoint2=10.119.168.231 ++ run_mysql 'SELECT @@hostname hostname;' '-h cross-site-source-haproxy -p33062 -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h cross-site-source-haproxy -p33062 -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ioAsjS2E0A ++++ mktemp +++ local LAST_ERR=/tmp/tmp.glJdkAO1JK +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ioAsjS2E0A +++ cat /tmp/tmp.glJdkAO1JK +++ rm /tmp/tmp.ioAsjS2E0A /tmp/tmp.glJdkAO1JK +++ return 0 ++ client_pod=pxc-client-6644d8898f-4dc42 ++ wait_pod pxc-client-6644d8898f-4dc42 ++ local pod=pxc-client-6644d8898f-4dc42 ++ local max_retry=480 ++ local ns= +++ echo pxc-client-6644d8898f-4dc42 +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok ++ set +o xtrace + source_primary=cross-site-source-pxc-0 ++ get_service_ip cross-site-source-pxc-0 ++ local service=cross-site-source-pxc-0 ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.YLgbINfnQs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MeQ7Y2pweA +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.YLgbINfnQs +++ cat /tmp/tmp.MeQ7Y2pweA +++ rm /tmp/tmp.YLgbINfnQs /tmp/tmp.MeQ7Y2pweA +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.l8M32OzRCE +++ mktemp ++ local LAST_ERR=/tmp/tmp.5LiHfCRRME ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.l8M32OzRCE ++ cat /tmp/tmp.5LiHfCRRME ++ rm /tmp/tmp.l8M32OzRCE /tmp/tmp.5LiHfCRRME ++ return 0 ++ return + source_primary_endpoint=10.119.165.152 + desc 'patch source cluster with replicationChannels settings' + set +o xtrace ----------------------------------------------------------------------------------- patch source cluster with replicationChannels settings ----------------------------------------------------------------------------------- + kubectl_bin patch pxc cross-site-source --type=merge --patch '{"spec": {"pxc":{"replicationChannels": [{"name":"source_to_replica", "isSource": true}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.GsVMLHdznG ++ mktemp + local LAST_ERR=/tmp/tmp.i0L5pJCxfI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-source --type=merge --patch '{"spec": {"pxc":{"replicationChannels": [{"name":"source_to_replica", "isSource": true}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GsVMLHdznG perconaxtradbcluster.pxc.percona.com/cross-site-source patched + cat /tmp/tmp.i0L5pJCxfI + rm /tmp/tmp.GsVMLHdznG /tmp/tmp.i0L5pJCxfI + return 0 + desc 'patch main cluster secrets with replication user' + set +o xtrace ----------------------------------------------------------------------------------- patch main cluster secrets with replication user ----------------------------------------------------------------------------------- ++ echo -n new_password ++ base64 + kubectl_bin patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.Wuubv4FmRZ ++ mktemp + local LAST_ERR=/tmp/tmp.b6VZ71G8He + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Wuubv4FmRZ secret/my-cluster-secrets patched + cat /tmp/tmp.b6VZ71G8He + rm /tmp/tmp.Wuubv4FmRZ /tmp/tmp.b6VZ71G8He + return 0 + sleep 15 + wait_cluster_consistency cross-site-source 3 2 + local cluster_name=cross-site-source + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KfbdHu0Fx4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.8EgNmuGyw3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KfbdHu0Fx4 ++ cat /tmp/tmp.8EgNmuGyw3 ++ rm /tmp/tmp.KfbdHu0Fx4 /tmp/tmp.8EgNmuGyw3 ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3pjnU8ZW0F +++ mktemp ++ local LAST_ERR=/tmp/tmp.x8JQXRrbwg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3pjnU8ZW0F ++ cat /tmp/tmp.x8JQXRrbwg ++ rm /tmp/tmp.3pjnU8ZW0F /tmp/tmp.x8JQXRrbwg ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.bdkvJajsuv ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ect1Zou2ZM +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.bdkvJajsuv +++++ cat /tmp/tmp.ect1Zou2ZM +++++ rm /tmp/tmp.bdkvJajsuv /tmp/tmp.ect1Zou2ZM +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-haproxy ++++ return +++ local cluster_proxy=cross-site-source-haproxy +++ echo haproxy ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UkanovBwMB +++ mktemp ++ local LAST_ERR=/tmp/tmp.5Ko4vwIJ1S ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UkanovBwMB ++ cat /tmp/tmp.5Ko4vwIJ1S ++ rm /tmp/tmp.UkanovBwMB /tmp/tmp.5Ko4vwIJ1S ++ return 0 + [[ 2 == \2 ]] + desc 'write data to source cluster' + set +o xtrace ----------------------------------------------------------------------------------- write data to source cluster ----------------------------------------------------------------------------------- + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testSourceReplica (id int PRIMARY KEY);' '-h 10.119.165.152 -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testSourceReplica (id int PRIMARY KEY);' + local 'uri=-h 10.119.165.152 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.16DxpqxKwq +++ mktemp ++ local LAST_ERR=/tmp/tmp.LDlSpf0Mtv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.16DxpqxKwq ++ cat /tmp/tmp.LDlSpf0Mtv ++ rm /tmp/tmp.16DxpqxKwq /tmp/tmp.LDlSpf0Mtv ++ return 0 + client_pod=pxc-client-6644d8898f-4dc42 + wait_pod pxc-client-6644d8898f-4dc42 + local pod=pxc-client-6644d8898f-4dc42 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-4dc42 + local container= + set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok + set +o xtrace + run_mysql 'INSERT myApp.testSourceReplica (id) VALUES (100700)' '-h 10.119.165.152 -uroot -proot_password' + local 'command=INSERT myApp.testSourceReplica (id) VALUES (100700)' + local 'uri=-h 10.119.165.152 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LjDsB2GAAp +++ mktemp ++ local LAST_ERR=/tmp/tmp.BM4xTey5GD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LjDsB2GAAp ++ cat /tmp/tmp.BM4xTey5GD ++ rm /tmp/tmp.LjDsB2GAAp /tmp/tmp.BM4xTey5GD ++ return 0 + client_pod=pxc-client-6644d8898f-4dc42 + wait_pod pxc-client-6644d8898f-4dc42 + local pod=pxc-client-6644d8898f-4dc42 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-4dc42 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok + set +o xtrace + kubectl_bin get secrets cross-site-source-ssl-internal -o yaml + yq 'del(.metadata)' - + yq '.metadata={"name": "cross-site-replica-ssl-internal"}' - ++ mktemp + local LAST_OUT=/tmp/tmp.mQJi2Huy2i ++ mktemp + local LAST_ERR=/tmp/tmp.YBLqebLSGv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get secrets cross-site-source-ssl-internal -o yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mQJi2Huy2i + cat /tmp/tmp.YBLqebLSGv + rm /tmp/tmp.mQJi2Huy2i /tmp/tmp.YBLqebLSGv + return 0 + desc 'take backup of source cluster' + set +o xtrace ----------------------------------------------------------------------------------- take backup of source cluster ----------------------------------------------------------------------------------- + run_backup cross-site-source backup-minio-source + local cluster=cross-site-source + local backup1=backup-minio-source + desc 'make backup backup-minio-source' + set +o xtrace ----------------------------------------------------------------------------------- make backup backup-minio-source ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/backup-minio-source.yml ++ mktemp + local LAST_OUT=/tmp/tmp.hJSq9jnjUC ++ mktemp + local LAST_ERR=/tmp/tmp.SrApQDZ7WJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/backup-minio-source.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hJSq9jnjUC perconaxtradbclusterbackup.pxc.percona.com/backup-minio-source created + cat /tmp/tmp.SrApQDZ7WJ + rm /tmp/tmp.hJSq9jnjUC /tmp/tmp.SrApQDZ7WJ + return 0 + wait_backup backup-minio-source + local backup=backup-minio-source + local status=Succeeded + set +o xtrace backup-minio-source..................Succeeded + desc 'create replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- create replica cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-replica-16033 0 + local namespace=cross-site-replica-16033 + local skip_clean_namespace=0 + [[ 1 == 1 ]] + [[ -z 0 ]] + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-replica-16033' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-replica-16033 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-replica-16033 ++ mktemp + local LAST_OUT=/tmp/tmp.6eBCmzspzh ++ mktemp + local LAST_ERR=/tmp/tmp.YmA0TdeeiC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-replica-16033 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-replica-16033 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-replica-16033 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.6eBCmzspzh + cat /tmp/tmp.YmA0TdeeiC Error from server (NotFound): namespaces "cross-site-replica-16033" not found + rm /tmp/tmp.6eBCmzspzh /tmp/tmp.YmA0TdeeiC + return 1 + : + wait_for_delete namespace/cross-site-replica-16033 + local res=namespace/cross-site-replica-16033 + echo -n 'namespace/cross-site-replica-16033 - ' namespace/cross-site-replica-16033 - + set +o xtrace Error from server (NotFound): namespaces "cross-site-replica-16033" not found + desc 'create namespace cross-site-replica-16033' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-replica-16033 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-replica-16033 ++ mktemp + local LAST_OUT=/tmp/tmp.PqIU6th9Ny ++ mktemp + local LAST_ERR=/tmp/tmp.X2zMh6UFmg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cross-site-replica-16033 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PqIU6th9Ny namespace/cross-site-replica-16033 created + cat /tmp/tmp.X2zMh6UFmg + rm /tmp/tmp.PqIU6th9Ny /tmp/tmp.X2zMh6UFmg + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.8mpGVjjBGR +++ mktemp ++ local LAST_ERR=/tmp/tmp.jIxvWtR2Wb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8mpGVjjBGR ++ cat /tmp/tmp.jIxvWtR2Wb ++ rm /tmp/tmp.8mpGVjjBGR /tmp/tmp.jIxvWtR2Wb ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3 --namespace=cross-site-replica-16033 ++ mktemp + local LAST_OUT=/tmp/tmp.3Ya5GIkaE4 ++ mktemp + local LAST_ERR=/tmp/tmp.wx9jFpziEq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3 --namespace=cross-site-replica-16033 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3Ya5GIkaE4 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3" modified. + cat /tmp/tmp.wx9jFpziEq + rm /tmp/tmp.3Ya5GIkaE4 /tmp/tmp.wx9jFpziEq + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.TZ81MlnbRh ++ mktemp + local LAST_ERR=/tmp/tmp.aIZhXWjOES + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TZ81MlnbRh customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.aIZhXWjOES + rm /tmp/tmp.TZ81MlnbRh /tmp/tmp.aIZhXWjOES + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.go8RFGIY9Y ++ mktemp + local LAST_ERR=/tmp/tmp.ukQGeCEK8c + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.go8RFGIY9Y clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.ukQGeCEK8c + rm /tmp/tmp.go8RFGIY9Y /tmp/tmp.ukQGeCEK8c + return 0 + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/deploy/cw-operator.yaml + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - + kubectl_bin apply -f - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' ++ mktemp + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + local LAST_OUT=/tmp/tmp.pXMuUvBl1i ++ mktemp + local LAST_ERR=/tmp/tmp.kUCQsUCg3P + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pXMuUvBl1i deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.kUCQsUCg3P + rm /tmp/tmp.pXMuUvBl1i /tmp/tmp.kUCQsUCg3P + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.PedghUdJLw ++ mktemp + local LAST_ERR=/tmp/tmp.sENlyYXMPF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PedghUdJLw pod/percona-xtradb-cluster-operator-675fc7d9f5-lht7c condition met + cat /tmp/tmp.sENlyYXMPF + rm /tmp/tmp.PedghUdJLw /tmp/tmp.sENlyYXMPF + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.5UKgRZpzm0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.opG9VIQzy0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5UKgRZpzm0 ++ cat /tmp/tmp.opG9VIQzy0 ++ rm /tmp/tmp.5UKgRZpzm0 /tmp/tmp.opG9VIQzy0 ++ return 0 + wait_pod percona-xtradb-cluster-operator-675fc7d9f5-jjk9g 480 pxc-operator + local pod=percona-xtradb-cluster-operator-675fc7d9f5-jjk9g + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-675fc7d9f5-jjk9g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-675fc7d9f5-jjk9g condition met percona-xtradb-cluster-operator-675fc7d9f5-jjk9g.Ok + sleep 3 + kubectl_bin apply -f /tmp/tmp.Oz5Ph9gkPr/replica-ssl-internal.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.tRET716GcG ++ mktemp + local LAST_ERR=/tmp/tmp.CHWiQXXEzo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /tmp/tmp.Oz5Ph9gkPr/replica-ssl-internal.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tRET716GcG secret/cross-site-replica-ssl-internal created + cat /tmp/tmp.CHWiQXXEzo + rm /tmp/tmp.tRET716GcG /tmp/tmp.CHWiQXXEzo + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.EqbCojohYg ++ mktemp + local LAST_ERR=/tmp/tmp.TDMcglwAot + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EqbCojohYg secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.TDMcglwAot + rm /tmp/tmp.EqbCojohYg /tmp/tmp.TDMcglwAot + return 0 + spinup_pxc cross-site-replica /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/cross-site-replica.yml + local cluster=cross-site-replica + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/cross-site-replica.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.q1stN7neDU ++ mktemp + local LAST_ERR=/tmp/tmp.uSSAnNCurn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.q1stN7neDU secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.uSSAnNCurn + rm /tmp/tmp.q1stN7neDU /tmp/tmp.uSSAnNCurn + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_OUT=/tmp/tmp.GiwWiv7ImV + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-21497~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.ThGZxfOaTO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GiwWiv7ImV deployment.apps/pxc-client created + cat /tmp/tmp.ThGZxfOaTO + rm /tmp/tmp.GiwWiv7ImV /tmp/tmp.ThGZxfOaTO + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/cross-site-replica.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/cross-site-replica.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/cross-site-replica.yml ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1743-51af0517#' + local LAST_OUT=/tmp/tmp.uFsViequXS + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-21497~ + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + local LAST_ERR=/tmp/tmp.xh1cNvCWi5 + local exit_status=0 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ seq 0 2 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uFsViequXS perconaxtradbcluster.pxc.percona.com/cross-site-replica created + cat /tmp/tmp.xh1cNvCWi5 + rm /tmp/tmp.uFsViequXS /tmp/tmp.xh1cNvCWi5 + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy cross-site-replica ++ local target_cluster=cross-site-replica +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TMDESk7kdN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uMf1EVLq42 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.TMDESk7kdN +++ cat /tmp/tmp.uMf1EVLq42 +++ rm /tmp/tmp.TMDESk7kdN /tmp/tmp.uMf1EVLq42 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo cross-site-replica-haproxy ++ return + local proxy=cross-site-replica-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-21497 ++ mktemp + local LAST_OUT=/tmp/tmp.8IQ5PsT9zY ++ mktemp + local LAST_ERR=/tmp/tmp.hFqJfF3psF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-21497 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-21497 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-21497 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.8IQ5PsT9zY + cat /tmp/tmp.hFqJfF3psF error: no matching resources found + rm /tmp/tmp.8IQ5PsT9zY /tmp/tmp.hFqJfF3psF + return 1 + true + wait_for_running cross-site-replica-haproxy 1 + local name=cross-site-replica-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-haproxy-0 480 + local pod=cross-site-replica-haproxy-0 + local max_retry=480 + local ns= ++ echo cross-site-replica-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace Error from server (NotFound): pods "cross-site-replica-haproxy-0" not found cross-site-replica-haproxy-0......................................Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running cross-site-replica-pxc 3 + local name=cross-site-replica-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-0 480 + local pod=cross-site-replica-pxc-0 + local max_retry=480 + local ns= ++ echo cross-site-replica-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/cross-site-replica-pxc-0 condition met cross-site-replica-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-1 480 + local pod=cross-site-replica-pxc-1 + local max_retry=480 + local ns= ++ echo cross-site-replica-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-replica-pxc-1 condition met cross-site-replica-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-2 480 + local pod=cross-site-replica-pxc-2 + local max_retry=480 + local ns= ++ echo cross-site-replica-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-replica-pxc-2 condition met cross-site-replica-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h cross-site-replica-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h cross-site-replica-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GWjo5iBhXU +++ mktemp ++ local LAST_ERR=/tmp/tmp.jGbhtUm0TA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GWjo5iBhXU ++ cat /tmp/tmp.jGbhtUm0TA ++ rm /tmp/tmp.GWjo5iBhXU /tmp/tmp.jGbhtUm0TA ++ return 0 + client_pod=pxc-client-6644d8898f-b4rr2 + wait_pod pxc-client-6644d8898f-b4rr2 + local pod=pxc-client-6644d8898f-b4rr2 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-b4rr2 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-b4rr2 condition met pxc-client-6644d8898f-b4rr2.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h cross-site-replica-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h cross-site-replica-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zPEJdSLp6o +++ mktemp ++ local LAST_ERR=/tmp/tmp.x5rHCGpPgu ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zPEJdSLp6o ++ cat /tmp/tmp.x5rHCGpPgu ++ rm /tmp/tmp.zPEJdSLp6o /tmp/tmp.x5rHCGpPgu ++ return 0 + client_pod=pxc-client-6644d8898f-b4rr2 + wait_pod pxc-client-6644d8898f-b4rr2 + local pod=pxc-client-6644d8898f-b4rr2 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-b4rr2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-b4rr2 condition met pxc-client-6644d8898f-b4rr2.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Y6nynQXyB4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.VuLnmHRv5W ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Y6nynQXyB4 ++ cat /tmp/tmp.VuLnmHRv5W ++ rm /tmp/tmp.Y6nynQXyB4 /tmp/tmp.VuLnmHRv5W ++ return 0 + client_pod=pxc-client-6644d8898f-b4rr2 + wait_pod pxc-client-6644d8898f-b4rr2 + local pod=pxc-client-6644d8898f-b4rr2 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-b4rr2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-b4rr2 condition met pxc-client-6644d8898f-b4rr2.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Oz5Ph9gkPr/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.Oz5Ph9gkPr/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.T7ExHsX4Y6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.l08NeDQ7fq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.T7ExHsX4Y6 ++ cat /tmp/tmp.l08NeDQ7fq ++ rm /tmp/tmp.T7ExHsX4Y6 /tmp/tmp.l08NeDQ7fq ++ return 0 + client_pod=pxc-client-6644d8898f-b4rr2 + wait_pod pxc-client-6644d8898f-b4rr2 + local pod=pxc-client-6644d8898f-b4rr2 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-b4rr2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-b4rr2 condition met pxc-client-6644d8898f-b4rr2.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Oz5Ph9gkPr/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.Oz5Ph9gkPr/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0SGvO0y0ln +++ mktemp ++ local LAST_ERR=/tmp/tmp.5TWV0CSbDn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0SGvO0y0ln ++ cat /tmp/tmp.5TWV0CSbDn ++ rm /tmp/tmp.0SGvO0y0ln /tmp/tmp.5TWV0CSbDn ++ return 0 + client_pod=pxc-client-6644d8898f-b4rr2 + wait_pod pxc-client-6644d8898f-b4rr2 + local pod=pxc-client-6644d8898f-b4rr2 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-b4rr2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-b4rr2 condition met pxc-client-6644d8898f-b4rr2.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Oz5Ph9gkPr/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.Oz5Ph9gkPr/select-1.sql ++ is_keyring_plugin_in_use cross-site-replica ++ local cluster=cross-site-replica ++ kubectl_bin exec -it cross-site-replica-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' +++ mktemp ++ local LAST_OUT=/tmp/tmp.B0JXgKCpk7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mIDGsFSNEU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it cross-site-replica-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.B0JXgKCpk7 ++ cat /tmp/tmp.mIDGsFSNEU Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.B0JXgKCpk7 /tmp/tmp.mIDGsFSNEU ++ return 0 + '[' '' ']' + sleep 60 + desc 'restore backup from source cluster' + set +o xtrace ----------------------------------------------------------------------------------- restore backup from source cluster ----------------------------------------------------------------------------------- ++ kubectl_bin get -n cross-site-21497 pxc-backup backup-minio-source -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.R2K9Rb0p3y +++ mktemp ++ local LAST_ERR=/tmp/tmp.xmLA13qPsI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get -n cross-site-21497 pxc-backup backup-minio-source -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.R2K9Rb0p3y ++ cat /tmp/tmp.xmLA13qPsI ++ rm /tmp/tmp.R2K9Rb0p3y /tmp/tmp.xmLA13qPsI ++ return 0 + destination=s3://operator-testing/cross-site-source-2024-07-10-13:28:51-full + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/restore-backup-minio.yml + /usr/bin/sed -e s~#cluster~cross-site-replica~ + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e s~#destination~s3://operator-testing/cross-site-source-2024-07-10-13:28:51-full~ + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-21497~ + local LAST_OUT=/tmp/tmp.f0iKSG3SzY ++ mktemp + local LAST_ERR=/tmp/tmp.hligEY3MZw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.f0iKSG3SzY perconaxtradbclusterrestore.pxc.percona.com/backup-minio created + cat /tmp/tmp.hligEY3MZw + rm /tmp/tmp.f0iKSG3SzY /tmp/tmp.hligEY3MZw + return 0 + wait_cluster_consistency cross-site-replica 3 2 + local cluster_name=cross-site-replica + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LYcXIrSgwI +++ mktemp ++ local LAST_ERR=/tmp/tmp.iukiJqFqCy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LYcXIrSgwI ++ cat /tmp/tmp.iukiJqFqCy ++ rm /tmp/tmp.LYcXIrSgwI /tmp/tmp.iukiJqFqCy ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VP0bkRxOef +++ mktemp ++ local LAST_ERR=/tmp/tmp.OWqCSs1YzF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VP0bkRxOef ++ cat /tmp/tmp.OWqCSs1YzF ++ rm /tmp/tmp.VP0bkRxOef /tmp/tmp.OWqCSs1YzF ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7QsJqzfV3j +++ mktemp ++ local LAST_ERR=/tmp/tmp.iR9zLebmag ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7QsJqzfV3j ++ cat /tmp/tmp.iR9zLebmag ++ rm /tmp/tmp.7QsJqzfV3j /tmp/tmp.iR9zLebmag ++ return 0 + [[ paused == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.m4nl9HrW9b +++ mktemp ++ local LAST_ERR=/tmp/tmp.Bvf7Uwo7hS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.m4nl9HrW9b ++ cat /tmp/tmp.Bvf7Uwo7hS ++ rm /tmp/tmp.m4nl9HrW9b /tmp/tmp.Bvf7Uwo7hS ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nd81562iUC +++ mktemp ++ local LAST_ERR=/tmp/tmp.xhdYmqwjGP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nd81562iUC ++ cat /tmp/tmp.xhdYmqwjGP ++ rm /tmp/tmp.nd81562iUC /tmp/tmp.xhdYmqwjGP ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.M3Fmg349iE +++ mktemp ++ local LAST_ERR=/tmp/tmp.PubIPhmJ9s ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.M3Fmg349iE ++ cat /tmp/tmp.PubIPhmJ9s ++ rm /tmp/tmp.M3Fmg349iE /tmp/tmp.PubIPhmJ9s ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7dqhm6ocfI +++ mktemp ++ local LAST_ERR=/tmp/tmp.CTIVpgqoiB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7dqhm6ocfI ++ cat /tmp/tmp.CTIVpgqoiB ++ rm /tmp/tmp.7dqhm6ocfI /tmp/tmp.CTIVpgqoiB ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Vjwpsk3Q54 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ftsWcl1U62 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Vjwpsk3Q54 ++ cat /tmp/tmp.ftsWcl1U62 ++ rm /tmp/tmp.Vjwpsk3Q54 /tmp/tmp.ftsWcl1U62 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AhJi9VNkgy +++ mktemp ++ local LAST_ERR=/tmp/tmp.iDjXImVxrp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AhJi9VNkgy ++ cat /tmp/tmp.iDjXImVxrp ++ rm /tmp/tmp.AhJi9VNkgy /tmp/tmp.iDjXImVxrp ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 8 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.77Rz7Db7sf +++ mktemp ++ local LAST_ERR=/tmp/tmp.VXSmRGPCmq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.77Rz7Db7sf ++ cat /tmp/tmp.VXSmRGPCmq ++ rm /tmp/tmp.77Rz7Db7sf /tmp/tmp.VXSmRGPCmq ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 9 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kqdp4mnUdc +++ mktemp ++ local LAST_ERR=/tmp/tmp.jUiTUZIUjz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kqdp4mnUdc ++ cat /tmp/tmp.jUiTUZIUjz ++ rm /tmp/tmp.kqdp4mnUdc /tmp/tmp.jUiTUZIUjz ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 10 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6oMOl9SmXs +++ mktemp ++ local LAST_ERR=/tmp/tmp.VtuklJR8NP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6oMOl9SmXs ++ cat /tmp/tmp.VtuklJR8NP ++ rm /tmp/tmp.6oMOl9SmXs /tmp/tmp.VtuklJR8NP ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 11 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.irhyu8lEJV +++ mktemp ++ local LAST_ERR=/tmp/tmp.y7rj6Vym0V ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.irhyu8lEJV ++ cat /tmp/tmp.y7rj6Vym0V ++ rm /tmp/tmp.irhyu8lEJV /tmp/tmp.y7rj6Vym0V ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2Sy1OVVRJA +++ mktemp ++ local LAST_ERR=/tmp/tmp.LqZY2xDfL9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2Sy1OVVRJA ++ cat /tmp/tmp.LqZY2xDfL9 ++ rm /tmp/tmp.2Sy1OVVRJA /tmp/tmp.LqZY2xDfL9 ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.FgQdD4m4SO ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.25ouniJN7S +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.FgQdD4m4SO +++++ cat /tmp/tmp.25ouniJN7S +++++ rm /tmp/tmp.FgQdD4m4SO /tmp/tmp.25ouniJN7S +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-haproxy ++++ return +++ local cluster_proxy=cross-site-replica-haproxy +++ echo haproxy ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mwapXKRsgG +++ mktemp ++ local LAST_ERR=/tmp/tmp.NKmOsXDQ6D ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mwapXKRsgG ++ cat /tmp/tmp.NKmOsXDQ6D ++ rm /tmp/tmp.mwapXKRsgG /tmp/tmp.NKmOsXDQ6D ++ return 0 + [[ 2 == \2 ]] + desc 'get replica cluster services endpoints' + set +o xtrace ----------------------------------------------------------------------------------- get replica cluster services endpoints ----------------------------------------------------------------------------------- ++ get_service_ip cross-site-replica-pxc-0 ++ local service=cross-site-replica-pxc-0 ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.aQB9YVheG6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.N79Es3Uhiq +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.aQB9YVheG6 +++ cat /tmp/tmp.N79Es3Uhiq +++ rm /tmp/tmp.aQB9YVheG6 /tmp/tmp.N79Es3Uhiq +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cD3YdtIb0t +++ mktemp ++ local LAST_ERR=/tmp/tmp.GkaUDqpEqE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cD3YdtIb0t ++ cat /tmp/tmp.GkaUDqpEqE ++ rm /tmp/tmp.cD3YdtIb0t /tmp/tmp.GkaUDqpEqE ++ return 0 ++ return + replica_endpoint0=10.119.161.234 ++ get_service_ip cross-site-replica-pxc-1 ++ local service=cross-site-replica-pxc-1 ++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DqQM9RjymV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YGjxiJyzZK +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.DqQM9RjymV +++ cat /tmp/tmp.YGjxiJyzZK +++ rm /tmp/tmp.DqQM9RjymV /tmp/tmp.YGjxiJyzZK +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.liulayveRs +++ mktemp ++ local LAST_ERR=/tmp/tmp.igabIRSXni ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.liulayveRs ++ cat /tmp/tmp.igabIRSXni ++ rm /tmp/tmp.liulayveRs /tmp/tmp.igabIRSXni ++ return 0 ++ return + replica_endpoint1=10.119.167.105 ++ get_service_ip cross-site-replica-pxc-2 ++ local service=cross-site-replica-pxc-2 ++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.q7WiXBeOPd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ID4bVP2Iu7 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.q7WiXBeOPd +++ cat /tmp/tmp.ID4bVP2Iu7 +++ rm /tmp/tmp.q7WiXBeOPd /tmp/tmp.ID4bVP2Iu7 +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Dt4iQSvY4m +++ mktemp ++ local LAST_ERR=/tmp/tmp.Q1p56UGPh4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Dt4iQSvY4m ++ cat /tmp/tmp.Q1p56UGPh4 ++ rm /tmp/tmp.Dt4iQSvY4m /tmp/tmp.Q1p56UGPh4 ++ return 0 ++ return + replica_endpoint2=10.119.172.83 ++ run_mysql 'SELECT @@hostname hostname;' '-h cross-site-replica-haproxy -p33062 -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h cross-site-replica-haproxy -p33062 -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yYrWZNVpVW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KtFjbs0ZLD +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.yYrWZNVpVW +++ cat /tmp/tmp.KtFjbs0ZLD +++ rm /tmp/tmp.yYrWZNVpVW /tmp/tmp.KtFjbs0ZLD +++ return 0 ++ client_pod=pxc-client-6644d8898f-b4rr2 ++ wait_pod pxc-client-6644d8898f-b4rr2 ++ local pod=pxc-client-6644d8898f-b4rr2 ++ local max_retry=480 ++ local ns= +++ echo pxc-client-6644d8898f-b4rr2 +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-6644d8898f-b4rr2 condition met pxc-client-6644d8898f-b4rr2.Ok ++ set +o xtrace + replica_primary=cross-site-replica-pxc-0 ++ get_service_ip cross-site-replica-pxc-0 ++ local service=cross-site-replica-pxc-0 ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HDRcS4lD1e ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kp7xPqVSvp +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.HDRcS4lD1e +++ cat /tmp/tmp.kp7xPqVSvp +++ rm /tmp/tmp.HDRcS4lD1e /tmp/tmp.kp7xPqVSvp +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xBXq9V27uZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.mjqcpXYlQb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xBXq9V27uZ ++ cat /tmp/tmp.mjqcpXYlQb ++ rm /tmp/tmp.xBXq9V27uZ /tmp/tmp.mjqcpXYlQb ++ return 0 ++ return + replica_primary_endpoint=10.119.161.234 + run_mysql 'DELETE FROM myApp.myApp WHERE id=100500' '-h 10.119.161.234 -uroot -proot_password' + local 'command=DELETE FROM myApp.myApp WHERE id=100500' + local 'uri=-h 10.119.161.234 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bpVTXQi7rd +++ mktemp ++ local LAST_ERR=/tmp/tmp.UZfxmhtKH2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bpVTXQi7rd ++ cat /tmp/tmp.UZfxmhtKH2 ++ rm /tmp/tmp.bpVTXQi7rd /tmp/tmp.UZfxmhtKH2 ++ return 0 + client_pod=pxc-client-6644d8898f-b4rr2 + wait_pod pxc-client-6644d8898f-b4rr2 + local pod=pxc-client-6644d8898f-b4rr2 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-b4rr2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-b4rr2 condition met pxc-client-6644d8898f-b4rr2.Ok + set +o xtrace + desc 'patch replica cluster with replicationChannels settings' + set +o xtrace ----------------------------------------------------------------------------------- patch replica cluster with replicationChannels settings ----------------------------------------------------------------------------------- + kubectl_bin patch pxc cross-site-replica --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "10.119.165.152", "port": 3306, "weight": 100},{"host": "10.119.168.71", "port": 3306, "weight": 100},{"host": "10.119.168.231", "port": 3306, "weight": 100}]}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.mOvE4JvtVN ++ mktemp + local LAST_ERR=/tmp/tmp.Xwxg6IG4qa + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-replica --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "10.119.165.152", "port": 3306, "weight": 100},{"host": "10.119.168.71", "port": 3306, "weight": 100},{"host": "10.119.168.231", "port": 3306, "weight": 100}]}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mOvE4JvtVN perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + cat /tmp/tmp.Xwxg6IG4qa + rm /tmp/tmp.mOvE4JvtVN /tmp/tmp.Xwxg6IG4qa + return 0 + sleep 40 + desc 'patch replica cluster secrets with replication user' + set +o xtrace ----------------------------------------------------------------------------------- patch replica cluster secrets with replication user ----------------------------------------------------------------------------------- ++ echo -n new_password ++ base64 + kubectl_bin patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.XEN1bScIqp ++ mktemp + local LAST_ERR=/tmp/tmp.0jbKgMTFxZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XEN1bScIqp secret/my-cluster-secrets patched + cat /tmp/tmp.0jbKgMTFxZ + rm /tmp/tmp.XEN1bScIqp /tmp/tmp.0jbKgMTFxZ + return 0 + sleep 15 + wait_cluster_consistency cross-site-replica 3 2 + local cluster_name=cross-site-replica + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lRgzJPwPgk +++ mktemp ++ local LAST_ERR=/tmp/tmp.hRDI6ht1cq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lRgzJPwPgk ++ cat /tmp/tmp.hRDI6ht1cq ++ rm /tmp/tmp.lRgzJPwPgk /tmp/tmp.hRDI6ht1cq ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ebFgz3aDFn +++ mktemp ++ local LAST_ERR=/tmp/tmp.2flqPzf9M0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ebFgz3aDFn ++ cat /tmp/tmp.2flqPzf9M0 ++ rm /tmp/tmp.ebFgz3aDFn /tmp/tmp.2flqPzf9M0 ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.WRwLV5djkM ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.j35XojEnvX +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.WRwLV5djkM +++++ cat /tmp/tmp.j35XojEnvX +++++ rm /tmp/tmp.WRwLV5djkM /tmp/tmp.j35XojEnvX +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-haproxy ++++ return +++ local cluster_proxy=cross-site-replica-haproxy +++ echo haproxy ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uB04jMVbDi +++ mktemp ++ local LAST_ERR=/tmp/tmp.RCfRmfWuLS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uB04jMVbDi ++ cat /tmp/tmp.RCfRmfWuLS ++ rm /tmp/tmp.uB04jMVbDi /tmp/tmp.RCfRmfWuLS ++ return 0 + [[ 2 == \2 ]] + desc 'Check replication works between source -> replica' + set +o xtrace ----------------------------------------------------------------------------------- Check replication works between source -> replica ----------------------------------------------------------------------------------- + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 10.119.161.234 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.119.161.234 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-2-80.sql ']' + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 10.119.161.234 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.119.161.234 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GQw5rSTwOH +++ mktemp ++ local LAST_ERR=/tmp/tmp.h4ZKc1fyaE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GQw5rSTwOH ++ cat /tmp/tmp.h4ZKc1fyaE ++ rm /tmp/tmp.GQw5rSTwOH /tmp/tmp.h4ZKc1fyaE ++ return 0 + client_pod=pxc-client-6644d8898f-b4rr2 + wait_pod pxc-client-6644d8898f-b4rr2 + local pod=pxc-client-6644d8898f-b4rr2 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-b4rr2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-b4rr2 condition met pxc-client-6644d8898f-b4rr2.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Oz5Ph9gkPr/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-2.sql /tmp/tmp.Oz5Ph9gkPr/select-2.sql + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 10.119.167.105 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.119.167.105 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-2-80.sql ']' + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 10.119.167.105 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.119.167.105 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EhhWMKThmk +++ mktemp ++ local LAST_ERR=/tmp/tmp.M32HMgUHnl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EhhWMKThmk ++ cat /tmp/tmp.M32HMgUHnl ++ rm /tmp/tmp.EhhWMKThmk /tmp/tmp.M32HMgUHnl ++ return 0 + client_pod=pxc-client-6644d8898f-b4rr2 + wait_pod pxc-client-6644d8898f-b4rr2 + local pod=pxc-client-6644d8898f-b4rr2 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-b4rr2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-b4rr2 condition met pxc-client-6644d8898f-b4rr2.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Oz5Ph9gkPr/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-2.sql /tmp/tmp.Oz5Ph9gkPr/select-2.sql + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 10.119.172.83 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.119.172.83 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-2-80.sql ']' + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 10.119.172.83 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.119.172.83 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XcNvNwVppk +++ mktemp ++ local LAST_ERR=/tmp/tmp.UWwuEwqaP7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XcNvNwVppk ++ cat /tmp/tmp.UWwuEwqaP7 ++ rm /tmp/tmp.XcNvNwVppk /tmp/tmp.UWwuEwqaP7 ++ return 0 + client_pod=pxc-client-6644d8898f-b4rr2 + wait_pod pxc-client-6644d8898f-b4rr2 + local pod=pxc-client-6644d8898f-b4rr2 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-b4rr2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-b4rr2 condition met pxc-client-6644d8898f-b4rr2.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Oz5Ph9gkPr/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-2.sql /tmp/tmp.Oz5Ph9gkPr/select-2.sql + run_backup cross-site-replica backup-minio-replica + local cluster=cross-site-replica + local backup1=backup-minio-replica + desc 'make backup backup-minio-replica' + set +o xtrace ----------------------------------------------------------------------------------- make backup backup-minio-replica ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/backup-minio-replica.yml ++ mktemp + local LAST_OUT=/tmp/tmp.jJRpCBQlyl ++ mktemp + local LAST_ERR=/tmp/tmp.zWTwoesi8V + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/backup-minio-replica.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jJRpCBQlyl perconaxtradbclusterbackup.pxc.percona.com/backup-minio-replica created + cat /tmp/tmp.zWTwoesi8V + rm /tmp/tmp.jJRpCBQlyl /tmp/tmp.zWTwoesi8V + return 0 + wait_backup backup-minio-replica + local backup=backup-minio-replica + local status=Succeeded + set +o xtrace backup-minio-replica.................Succeeded + desc 'Switch clusters over' + set +o xtrace ----------------------------------------------------------------------------------- Switch clusters over ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.vPFwOeERZN +++ mktemp ++ local LAST_ERR=/tmp/tmp.b13HZ9WnWY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vPFwOeERZN ++ cat /tmp/tmp.b13HZ9WnWY ++ rm /tmp/tmp.vPFwOeERZN /tmp/tmp.b13HZ9WnWY ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3 --namespace=cross-site-21497 ++ mktemp + local LAST_OUT=/tmp/tmp.3K08QO1lw7 ++ mktemp + local LAST_ERR=/tmp/tmp.KOEGXJ5tPv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3 --namespace=cross-site-21497 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3K08QO1lw7 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3" modified. + cat /tmp/tmp.KOEGXJ5tPv + rm /tmp/tmp.3K08QO1lw7 /tmp/tmp.KOEGXJ5tPv + return 0 + desc 'rebuild source cluster' + set +o xtrace ----------------------------------------------------------------------------------- rebuild source cluster ----------------------------------------------------------------------------------- ++ kubectl_bin get -n cross-site-replica-16033 pxc-backup backup-minio-replica -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.y2oGBqGDVG +++ mktemp ++ local LAST_ERR=/tmp/tmp.F6lyK5vqM8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get -n cross-site-replica-16033 pxc-backup backup-minio-replica -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.y2oGBqGDVG ++ cat /tmp/tmp.F6lyK5vqM8 ++ rm /tmp/tmp.y2oGBqGDVG /tmp/tmp.F6lyK5vqM8 ++ return 0 + destination=s3://operator-testing/cross-site-replica-2024-07-10-13:43:13-full + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/conf/restore-backup-minio.yml + /usr/bin/sed -e s~#destination~s3://operator-testing/cross-site-replica-2024-07-10-13:43:13-full~ + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-21497~ + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e s~#cluster~cross-site-source~ + local LAST_OUT=/tmp/tmp.l0ivMUUDYr ++ mktemp + local LAST_ERR=/tmp/tmp.D3QIozFinr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.l0ivMUUDYr perconaxtradbclusterrestore.pxc.percona.com/backup-minio created + cat /tmp/tmp.D3QIozFinr + rm /tmp/tmp.l0ivMUUDYr /tmp/tmp.D3QIozFinr + return 0 + wait_cluster_consistency cross-site-source 3 2 + local cluster_name=cross-site-source + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ICZZLJdOoE +++ mktemp ++ local LAST_ERR=/tmp/tmp.gUyFGFOhvA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ICZZLJdOoE ++ cat /tmp/tmp.gUyFGFOhvA ++ rm /tmp/tmp.ICZZLJdOoE /tmp/tmp.gUyFGFOhvA ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RvBNIw1wfW +++ mktemp ++ local LAST_ERR=/tmp/tmp.9k4UbRtRNR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RvBNIw1wfW ++ cat /tmp/tmp.9k4UbRtRNR ++ rm /tmp/tmp.RvBNIw1wfW /tmp/tmp.9k4UbRtRNR ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5lsYxTtGAR +++ mktemp ++ local LAST_ERR=/tmp/tmp.bNBZRV6qmg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5lsYxTtGAR ++ cat /tmp/tmp.bNBZRV6qmg ++ rm /tmp/tmp.5lsYxTtGAR /tmp/tmp.bNBZRV6qmg ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f6SXnsr2ep +++ mktemp ++ local LAST_ERR=/tmp/tmp.5nigJe5gBR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.f6SXnsr2ep ++ cat /tmp/tmp.5nigJe5gBR ++ rm /tmp/tmp.f6SXnsr2ep /tmp/tmp.5nigJe5gBR ++ return 0 + [[ paused == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q6R1a1n508 +++ mktemp ++ local LAST_ERR=/tmp/tmp.QQd4IA9UD7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.q6R1a1n508 ++ cat /tmp/tmp.QQd4IA9UD7 ++ rm /tmp/tmp.q6R1a1n508 /tmp/tmp.QQd4IA9UD7 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z8CkHJTmUP +++ mktemp ++ local LAST_ERR=/tmp/tmp.jhLBzM3u0H ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Z8CkHJTmUP ++ cat /tmp/tmp.jhLBzM3u0H ++ rm /tmp/tmp.Z8CkHJTmUP /tmp/tmp.jhLBzM3u0H ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4hS4itGLVv +++ mktemp ++ local LAST_ERR=/tmp/tmp.ahOl5ufhct ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4hS4itGLVv ++ cat /tmp/tmp.ahOl5ufhct ++ rm /tmp/tmp.4hS4itGLVv /tmp/tmp.ahOl5ufhct ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XwY1frPTuH +++ mktemp ++ local LAST_ERR=/tmp/tmp.sCXmyTHpmD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XwY1frPTuH ++ cat /tmp/tmp.sCXmyTHpmD ++ rm /tmp/tmp.XwY1frPTuH /tmp/tmp.sCXmyTHpmD ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6JZXfgSNhd +++ mktemp ++ local LAST_ERR=/tmp/tmp.Kjx7H3qp3E ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6JZXfgSNhd ++ cat /tmp/tmp.Kjx7H3qp3E ++ rm /tmp/tmp.6JZXfgSNhd /tmp/tmp.Kjx7H3qp3E ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 8 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wpgcuDWozm +++ mktemp ++ local LAST_ERR=/tmp/tmp.dGSAzSrawd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wpgcuDWozm ++ cat /tmp/tmp.dGSAzSrawd ++ rm /tmp/tmp.wpgcuDWozm /tmp/tmp.dGSAzSrawd ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 9 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uqiMSDDUjo +++ mktemp ++ local LAST_ERR=/tmp/tmp.B4nHL0e05a ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uqiMSDDUjo ++ cat /tmp/tmp.B4nHL0e05a ++ rm /tmp/tmp.uqiMSDDUjo /tmp/tmp.B4nHL0e05a ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 10 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OO7M9WkjLi +++ mktemp ++ local LAST_ERR=/tmp/tmp.kvV7f3PX9k ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OO7M9WkjLi ++ cat /tmp/tmp.kvV7f3PX9k ++ rm /tmp/tmp.OO7M9WkjLi /tmp/tmp.kvV7f3PX9k ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 11 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ir2QXuFxcf +++ mktemp ++ local LAST_ERR=/tmp/tmp.hLVhTn7ywa ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ir2QXuFxcf ++ cat /tmp/tmp.hLVhTn7ywa ++ rm /tmp/tmp.ir2QXuFxcf /tmp/tmp.hLVhTn7ywa ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.agkHK9kGpI +++ mktemp ++ local LAST_ERR=/tmp/tmp.ndwNelrZM7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.agkHK9kGpI ++ cat /tmp/tmp.ndwNelrZM7 ++ rm /tmp/tmp.agkHK9kGpI /tmp/tmp.ndwNelrZM7 ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.67uPwPSb5j ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Uy2tooYvI2 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.67uPwPSb5j +++++ cat /tmp/tmp.Uy2tooYvI2 +++++ rm /tmp/tmp.67uPwPSb5j /tmp/tmp.Uy2tooYvI2 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-haproxy ++++ return +++ local cluster_proxy=cross-site-source-haproxy +++ echo haproxy ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.anfMC4b6wP +++ mktemp ++ local LAST_ERR=/tmp/tmp.tdlPj97Ztj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.anfMC4b6wP ++ cat /tmp/tmp.tdlPj97Ztj ++ rm /tmp/tmp.anfMC4b6wP /tmp/tmp.tdlPj97Ztj ++ return 0 + [[ 2 == \2 ]] + run_mysql 'DELETE FROM myApp.myApp WHERE id=100500' '-h 10.119.165.152 -uroot -proot_password' + local 'command=DELETE FROM myApp.myApp WHERE id=100500' + local 'uri=-h 10.119.165.152 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LCZHdL7oln +++ mktemp ++ local LAST_ERR=/tmp/tmp.iGsIqHhtUT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LCZHdL7oln ++ cat /tmp/tmp.iGsIqHhtUT ++ rm /tmp/tmp.LCZHdL7oln /tmp/tmp.iGsIqHhtUT ++ return 0 + client_pod=pxc-client-6644d8898f-4dc42 + wait_pod pxc-client-6644d8898f-4dc42 + local pod=pxc-client-6644d8898f-4dc42 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-4dc42 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok + set +o xtrace + run_mysql 'DELETE FROM myApp.testSourceReplica WHERE id=100700' '-h 10.119.165.152 -uroot -proot_password' + local 'command=DELETE FROM myApp.testSourceReplica WHERE id=100700' + local 'uri=-h 10.119.165.152 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ldDzKfSEMf +++ mktemp ++ local LAST_ERR=/tmp/tmp.2tSwjjhhsr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ldDzKfSEMf ++ cat /tmp/tmp.2tSwjjhhsr ++ rm /tmp/tmp.ldDzKfSEMf /tmp/tmp.2tSwjjhhsr ++ return 0 + client_pod=pxc-client-6644d8898f-4dc42 + wait_pod pxc-client-6644d8898f-4dc42 + local pod=pxc-client-6644d8898f-4dc42 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6644d8898f-4dc42 ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok + set +o xtrace + desc 'configure old replica as source' + set +o xtrace ----------------------------------------------------------------------------------- configure old replica as source ----------------------------------------------------------------------------------- + kubectl patch pxc cross-site-replica -n cross-site-replica-16033 --type=json '-p=[{'\''op'\'': '\''remove'\'', '\''path'\'': '\''/spec/pxc/replicationChannels/0'\''}]' perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + kubectl_bin patch pxc cross-site-replica -n cross-site-replica-16033 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": true}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.PKJMutJYW4 ++ mktemp + local LAST_ERR=/tmp/tmp.vTyXZQd6Y7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-replica -n cross-site-replica-16033 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": true}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PKJMutJYW4 perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + cat /tmp/tmp.vTyXZQd6Y7 + rm /tmp/tmp.PKJMutJYW4 /tmp/tmp.vTyXZQd6Y7 + return 0 + desc 'configure old source as replica' + set +o xtrace ----------------------------------------------------------------------------------- configure old source as replica ----------------------------------------------------------------------------------- + kubectl patch pxc cross-site-source -n cross-site-21497 --type=json '-p=[{'\''op'\'': '\''remove'\'', '\''path'\'': '\''/spec/pxc/replicationChannels/0'\''}]' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + kubectl_bin patch pxc cross-site-source -n cross-site-21497 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "10.119.161.234", "port": 3306, "weight": 100},{"host": "10.119.167.105", "port": 3306, "weight": 100},{"host": "10.119.172.83", "port": 3306, "weight": 100}]}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.r54GPckTKs ++ mktemp + local LAST_ERR=/tmp/tmp.khao2drsCj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-source -n cross-site-21497 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "10.119.161.234", "port": 3306, "weight": 100},{"host": "10.119.167.105", "port": 3306, "weight": 100},{"host": "10.119.172.83", "port": 3306, "weight": 100}]}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.r54GPckTKs perconaxtradbcluster.pxc.percona.com/cross-site-source patched + cat /tmp/tmp.khao2drsCj + rm /tmp/tmp.r54GPckTKs /tmp/tmp.khao2drsCj + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.GpkZ8JkY9F +++ mktemp ++ local LAST_ERR=/tmp/tmp.oRtJO20HxQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GpkZ8JkY9F ++ cat /tmp/tmp.oRtJO20HxQ ++ rm /tmp/tmp.GpkZ8JkY9F /tmp/tmp.oRtJO20HxQ ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3 --namespace=cross-site-replica-16033 ++ mktemp + local LAST_OUT=/tmp/tmp.mrgugmiKkx ++ mktemp + local LAST_ERR=/tmp/tmp.f017uS0mEO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3 --namespace=cross-site-replica-16033 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mrgugmiKkx Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3" modified. + cat /tmp/tmp.f017uS0mEO + rm /tmp/tmp.mrgugmiKkx /tmp/tmp.f017uS0mEO + return 0 + desc 'Write data to replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- Write data to replica cluster ----------------------------------------------------------------------------------- + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testReplicaSource (id int PRIMARY KEY);' '-h 10.119.161.234 -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testReplicaSource (id int PRIMARY KEY);' + local 'uri=-h 10.119.161.234 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GsZPtwNjCa +++ mktemp ++ local LAST_ERR=/tmp/tmp.DFcKLMJheQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GsZPtwNjCa ++ cat /tmp/tmp.DFcKLMJheQ ++ rm /tmp/tmp.GsZPtwNjCa /tmp/tmp.DFcKLMJheQ ++ return 0 + client_pod=pxc-client-6644d8898f-b4rr2 + wait_pod pxc-client-6644d8898f-b4rr2 + local pod=pxc-client-6644d8898f-b4rr2 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-b4rr2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-b4rr2 condition met pxc-client-6644d8898f-b4rr2.Ok + set +o xtrace + run_mysql 'INSERT myApp.testReplicaSource (id) VALUES (100800)' '-h 10.119.161.234 -uroot -proot_password' + local 'command=INSERT myApp.testReplicaSource (id) VALUES (100800)' + local 'uri=-h 10.119.161.234 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qAwhC3fZrU +++ mktemp ++ local LAST_ERR=/tmp/tmp.AxsVJifyHq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qAwhC3fZrU ++ cat /tmp/tmp.AxsVJifyHq ++ rm /tmp/tmp.qAwhC3fZrU /tmp/tmp.AxsVJifyHq ++ return 0 + client_pod=pxc-client-6644d8898f-b4rr2 + wait_pod pxc-client-6644d8898f-b4rr2 + local pod=pxc-client-6644d8898f-b4rr2 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-b4rr2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-b4rr2 condition met pxc-client-6644d8898f-b4rr2.Ok + set +o xtrace + sleep 15 + desc 'Check replication works between replica -> source' + set +o xtrace ----------------------------------------------------------------------------------- Check replication works between replica -> source ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.a2At4s0FZx +++ mktemp ++ local LAST_ERR=/tmp/tmp.9LfhMY97Tv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.a2At4s0FZx ++ cat /tmp/tmp.9LfhMY97Tv ++ rm /tmp/tmp.a2At4s0FZx /tmp/tmp.9LfhMY97Tv ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3 --namespace=cross-site-21497 ++ mktemp + local LAST_OUT=/tmp/tmp.DXkVXSycHC ++ mktemp + local LAST_ERR=/tmp/tmp.9jrfg6uP99 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3 --namespace=cross-site-21497 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DXkVXSycHC Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1743-51af0517-14-cluster3" modified. + cat /tmp/tmp.9jrfg6uP99 + rm /tmp/tmp.DXkVXSycHC /tmp/tmp.9jrfg6uP99 + return 0 + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 10.119.165.152 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.119.165.152 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-3-80.sql ']' + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 10.119.165.152 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.119.165.152 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dLxGV8qELK +++ mktemp ++ local LAST_ERR=/tmp/tmp.8nx0MTQLK6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dLxGV8qELK ++ cat /tmp/tmp.8nx0MTQLK6 ++ rm /tmp/tmp.dLxGV8qELK /tmp/tmp.8nx0MTQLK6 ++ return 0 + client_pod=pxc-client-6644d8898f-4dc42 + wait_pod pxc-client-6644d8898f-4dc42 + local pod=pxc-client-6644d8898f-4dc42 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-4dc42 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Oz5Ph9gkPr/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-3.sql /tmp/tmp.Oz5Ph9gkPr/select-3.sql + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 10.119.168.71 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.119.168.71 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-3-80.sql ']' + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 10.119.168.71 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.119.168.71 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A9zyOnnwQu +++ mktemp ++ local LAST_ERR=/tmp/tmp.OcGgG5twKe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.A9zyOnnwQu ++ cat /tmp/tmp.OcGgG5twKe ++ rm /tmp/tmp.A9zyOnnwQu /tmp/tmp.OcGgG5twKe ++ return 0 + client_pod=pxc-client-6644d8898f-4dc42 + wait_pod pxc-client-6644d8898f-4dc42 + local pod=pxc-client-6644d8898f-4dc42 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-4dc42 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Oz5Ph9gkPr/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-3.sql /tmp/tmp.Oz5Ph9gkPr/select-3.sql + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 10.119.168.231 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.119.168.231 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-3-80.sql ']' + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 10.119.168.231 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.119.168.231 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iCjPidi8M0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.pOlm7tfhyR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iCjPidi8M0 ++ cat /tmp/tmp.pOlm7tfhyR ++ rm /tmp/tmp.iCjPidi8M0 /tmp/tmp.pOlm7tfhyR ++ return 0 + client_pod=pxc-client-6644d8898f-4dc42 + wait_pod pxc-client-6644d8898f-4dc42 + local pod=pxc-client-6644d8898f-4dc42 + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-4dc42 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-4dc42 condition met pxc-client-6644d8898f-4dc42.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Oz5Ph9gkPr/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1743/e2e-tests/cross-site/compare/select-3.sql /tmp/tmp.Oz5Ph9gkPr/select-3.sql + destroy cross-site-21497 true + local namespace=cross-site-21497 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v level=info + grep -v 'get backup status: Job.batch' + sort -u + tee /tmp/tmp.Oz5Ph9gkPr/operator.log + grep -v 'the object has been modified' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.r0uiAQOUta +++ mktemp ++ local LAST_ERR=/tmp/tmp.JwwVxxPsD1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.r0uiAQOUta ++ cat /tmp/tmp.JwwVxxPsD1 ++ rm /tmp/tmp.r0uiAQOUta /tmp/tmp.JwwVxxPsD1 ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-675fc7d9f5-jjk9g ++ mktemp + local LAST_OUT=/tmp/tmp.Mbp2hjgdK3 ++ mktemp + local LAST_ERR=/tmp/tmp.a6jAFCsL29 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-675fc7d9f5-jjk9g + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Mbp2hjgdK3 + cat /tmp/tmp.a6jAFCsL29 + rm /tmp/tmp.Mbp2hjgdK3 /tmp/tmp.a6jAFCsL29 + return 0 2024-07-10T13:19:27.836Z INFO setup Manager starting up {"gitCommit": "51af051703ea68e6367821ec3068826842ab5c9d", "gitBranch": "PR-1743-51af0517", "buildTime": "2024-07-10T11:38:48Z", "goVersion": "go1.22.5", "os": "linux", "arch": "amd64"} 2024-07-10T13:19:27.836Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.26.15-gke.1469001"} 2024-07-10T13:19:27.837Z INFO setup Registering Components. 2024-07-10T13:19:31.113Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2024-07-10T13:19:31.116Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2024-07-10T13:19:31.116Z INFO controller-runtime.metrics Starting metrics server 2024-07-10T13:19:31.116Z INFO controller-runtime.webhook Starting webhook server 2024-07-10T13:19:31.116Z INFO setup Starting the Cmd. 2024-07-10T13:19:31.116Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2024-07-10T13:19:31.117Z INFO controller-runtime.certwatcher Starting certificate watcher 2024-07-10T13:19:31.117Z INFO controller-runtime.certwatcher Updated current TLS certificate 2024-07-10T13:19:31.117Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2024-07-10T13:19:31.218Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2024-07-10T13:19:31.233Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2024-07-10T13:19:31.234Z DEBUG events percona-xtradb-cluster-operator-675fc7d9f5-jjk9g_c7d207b8-c6f2-489d-a0b8-6d0b27ac5b96 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"0c84f1d1-331c-447b-947d-d4326fea30e6","apiVersion":"coordination.k8s.io/v1","resourceVersion":"48182"}, "reason": "LeaderElection"} 2024-07-10T13:19:31.234Z INFO Starting Controller {"controller": "pxc-controller"} 2024-07-10T13:19:31.234Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2024-07-10T13:19:31.234Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"} 2024-07-10T13:19:31.234Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2024-07-10T13:19:31.235Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2024-07-10T13:19:31.235Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2024-07-10T13:19:31.446Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2024-07-10T13:19:31.446Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2024-07-10T13:19:31.446Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2024-07-10T13:21:47.899Z INFO Set CR version {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "version": "1.15.0"} 2024-07-10T13:22:02.230Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-pxc"} 2024-07-10T13:22:02.354Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-pxc"} 2024-07-10T13:22:02.438Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-pxc-unready"} 2024-07-10T13:22:02.552Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-pxc-0"} 2024-07-10T13:22:02.759Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-pxc-1"} 2024-07-10T13:22:02.925Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-pxc-2"} 2024-07-10T13:22:02.989Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-haproxy"} 2024-07-10T13:22:03.028Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-haproxy"} 2024-07-10T13:22:03.064Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-haproxy-replicas"} 2024-07-10T13:22:03.156Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 5b91f771-be74-47e4-aaa7-7f0007fd41d1 2024-07-10T13:22:03.356Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "044f02d3-2db3-4dd2-aaff-f26b8b178e48", "object": "cross-site-source-pxc"} 2024-07-10T13:22:03.393Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "044f02d3-2db3-4dd2-aaff-f26b8b178e48", "object": "cross-site-source-haproxy"} 2024-07-10T13:23:24.954Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a", "user": "operator"} 2024-07-10T13:23:24.997Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a", "user": "monitor"} 2024-07-10T13:23:25.094Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a"} 2024-07-10T13:23:25.130Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a"} 2024-07-10T13:23:25.172Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a", "user": "xtrabackup"} 2024-07-10T13:23:25.250Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a"} 2024-07-10T13:23:25.317Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a", "user": "replication"} 2024-07-10T13:25:46.297Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ee3c7958-b364-45eb-bfd2-ef70b1773755", "user": "root"} 2024-07-10T13:25:46.564Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ee3c7958-b364-45eb-bfd2-ef70b1773755", "new version": "8.0.36-28.1"} 2024-07-10T13:28:14.411Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "17dbfb82-8ecd-45e8-8e1b-21a9b5e1f239", "user": "replication"} 2024-07-10T13:28:14.447Z INFO Password updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "17dbfb82-8ecd-45e8-8e1b-21a9b5e1f239", "user": "replication"} 2024-07-10T13:28:14.455Z INFO MySQL init secret created {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "17dbfb82-8ecd-45e8-8e1b-21a9b5e1f239", "secret": "cross-site-source-mysql-init", "user": "replication"} 2024-07-10T13:28:14.467Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "17dbfb82-8ecd-45e8-8e1b-21a9b5e1f239", "user": "replication"} 2024-07-10T13:28:14.495Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "17dbfb82-8ecd-45e8-8e1b-21a9b5e1f239", "user": "replication"} 2024-07-10T13:28:52.038Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "cross-site-21497", "name": "backup-minio-source", "reconcileID": "0ee0c548-206c-4d42-9d62-419433b2c0d0", "Namespace": "cross-site-21497", "Name": "xb-backup-minio-source"} 2024-07-10T13:30:53.681Z INFO Set CR version {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "version": "1.15.0"} 2024-07-10T13:31:01.424Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-pxc"} 2024-07-10T13:31:01.478Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-haproxy"} 2024-07-10T13:31:01.509Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-pxc"} 2024-07-10T13:31:01.578Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-pxc-unready"} 2024-07-10T13:31:01.751Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-pxc-0"} 2024-07-10T13:31:01.984Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-pxc-1"} 2024-07-10T13:31:02.071Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-pxc-2"} 2024-07-10T13:31:02.150Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-haproxy"} 2024-07-10T13:31:02.188Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-haproxy"} 2024-07-10T13:31:02.228Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-haproxy-replicas"} 2024-07-10T13:31:02.268Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: f279ae47-e06d-4fe2-a4e0-f03232971010 2024-07-10T13:31:02.727Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "261dc436-45a0-497f-85c4-a1fa51eea25f", "object": "cross-site-replica-pxc"} 2024-07-10T13:32:18.854Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0", "user": "operator"} 2024-07-10T13:32:18.895Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0", "user": "monitor"} 2024-07-10T13:32:18.987Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0"} 2024-07-10T13:32:19.023Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0"} 2024-07-10T13:32:19.059Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0", "user": "xtrabackup"} 2024-07-10T13:32:19.349Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0"} 2024-07-10T13:32:19.601Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0", "user": "replication"} 2024-07-10T13:34:53.296Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f12c590b-e439-45ca-930a-834047119a57", "user": "root"} 2024-07-10T13:34:53.563Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f12c590b-e439-45ca-930a-834047119a57", "new version": "8.0.36-28.1"} 2024-07-10T13:37:07.388Z INFO backup restore request {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-16033", "name": "backup-minio", "reconcileID": "3ef59fcb-535b-44a1-b0ce-03ca2d47ee32"} 2024-07-10T13:37:07.438Z INFO stopping cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-16033", "name": "backup-minio", "reconcileID": "3ef59fcb-535b-44a1-b0ce-03ca2d47ee32", "cluster": "cross-site-replica"} 2024-07-10T13:37:07.568Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "14e470ec-5823-499b-92d7-f3a14c597881", "object": "cross-site-replica-pxc"} 2024-07-10T13:37:07.674Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "14e470ec-5823-499b-92d7-f3a14c597881", "object": "cross-site-replica-haproxy"} 2024-07-10T13:37:53.577Z INFO starting restore {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-16033", "name": "backup-minio", "reconcileID": "3ef59fcb-535b-44a1-b0ce-03ca2d47ee32", "cluster": "cross-site-replica", "backup": ""} 2024-07-10T13:38:12.665Z INFO starting cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-16033", "name": "backup-minio", "reconcileID": "3ef59fcb-535b-44a1-b0ce-03ca2d47ee32", "cluster": "cross-site-replica"} 2024-07-10T13:38:12.740Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "74a31a7e-c5d3-4cb9-a386-801db39b5aac", "object": "cross-site-replica-pxc"} 2024-07-10T13:38:12.807Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "74a31a7e-c5d3-4cb9-a386-801db39b5aac", "object": "cross-site-replica-haproxy"} 2024-07-10T13:41:16.870Z INFO You can view xtrabackup log: 2024-07-10T13:41:47.911Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "00e9e3ee-e1ab-4853-a70c-ee70eace116f", "pod": "cross-site-replica-pxc-2"} 2024-07-10T13:41:47.930Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "00e9e3ee-e1ab-4853-a70c-ee70eace116f", "pod": "cross-site-replica-pxc-0"} 2024-07-10T13:41:47.951Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "00e9e3ee-e1ab-4853-a70c-ee70eace116f", "pod": "cross-site-replica-pxc-1"} 2024-07-10T13:41:48.013Z INFO Replication pod has changed {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "00e9e3ee-e1ab-4853-a70c-ee70eace116f", "new replication pod": "cross-site-replica-pxc-0"} 2024-07-10T13:41:49.959Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "4bb9ecfa-7d9a-4ae1-ac34-ce66f68624e4", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:41:53.946Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "8529ecd8-065d-4112-91c9-e943da37060e", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:41:59.309Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "3f6b6c30-fa3b-4d52-b552-2380e94da519", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:42:05.432Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "91e5ecb0-9a77-4527-a88e-916567946568", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:42:10.910Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "b12ef1aa-9ef1-4b7a-95fb-c5318c836035", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:42:16.257Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "1d4f177e-e8bd-4937-a71c-dd9e52f8dca8", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:42:21.569Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "d93f570a-541b-4039-a30c-625df3b093c2", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:42:26.964Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ea02bbbe-5098-4693-90d6-d78a7d836692", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:42:32.185Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "dd37927e-a02d-48b2-9325-b30c8a315c60", "user": "replication"} 2024-07-10T13:42:32.234Z INFO Password updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "dd37927e-a02d-48b2-9325-b30c8a315c60", "user": "replication"} 2024-07-10T13:42:32.241Z INFO MySQL init secret created {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "dd37927e-a02d-48b2-9325-b30c8a315c60", "secret": "cross-site-replica-mysql-init", "user": "replication"} 2024-07-10T13:42:32.255Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "dd37927e-a02d-48b2-9325-b30c8a315c60", "user": "replication"} 2024-07-10T13:42:32.285Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "dd37927e-a02d-48b2-9325-b30c8a315c60", "user": "replication"} 2024-07-10T13:43:13.366Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "cross-site-replica-16033", "name": "backup-minio-replica", "reconcileID": "4a375727-d790-49f0-bd18-f662d445b22a", "Namespace": "cross-site-replica-16033", "Name": "xb-backup-minio-replica"} 2024-07-10T13:43:48.128Z INFO backup restore request {"controller": "pxcrestore-controller", "namespace": "cross-site-21497", "name": "backup-minio", "reconcileID": "e62b38fd-b74d-432f-9adf-312cf9946b09"} 2024-07-10T13:43:48.209Z INFO stopping cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-21497", "name": "backup-minio", "reconcileID": "e62b38fd-b74d-432f-9adf-312cf9946b09", "cluster": "cross-site-source"} 2024-07-10T13:43:48.415Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "261580cc-dc4e-4183-b647-52768e730334", "object": "cross-site-source-pxc"} 2024-07-10T13:43:48.515Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "261580cc-dc4e-4183-b647-52768e730334", "object": "cross-site-source-haproxy"} 2024-07-10T13:44:10.416Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "78f12912-32e7-4000-bdfb-856267406fbc", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:15.821Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "36d0e170-4930-4076-8193-0902b78b41ac", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:21.143Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "c1ac7902-5c2f-42df-93db-88c1ab068114", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:26.570Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "05f53885-47ba-477d-b686-cd0e7e5eee13", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:31.920Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "c779025d-5c6d-46a4-9a76-022eb46ab99f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:37.238Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "dcaa2b08-2d72-4f6d-a0b8-adf7eaef6c54", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:40.407Z INFO starting restore {"controller": "pxcrestore-controller", "namespace": "cross-site-21497", "name": "backup-minio", "reconcileID": "e62b38fd-b74d-432f-9adf-312cf9946b09", "cluster": "cross-site-source", "backup": ""} 2024-07-10T13:44:42.620Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "d2dc6447-6cdb-4d65-9880-5a4396eadb86", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:48.015Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "2a158491-89f5-4ffa-a67e-fdf89185cd77", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:53.332Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "e6ea218d-7843-4737-8522-8402156da593", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:58.728Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "c05c7052-ff96-4d07-81dd-541b0412bbf1", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:00.531Z INFO starting cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-21497", "name": "backup-minio", "reconcileID": "e62b38fd-b74d-432f-9adf-312cf9946b09", "cluster": "cross-site-source"} 2024-07-10T13:45:00.590Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "d0772ca0-6153-46b9-bac1-c2117fed6e0d", "object": "cross-site-source-pxc"} 2024-07-10T13:45:00.678Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "d0772ca0-6153-46b9-bac1-c2117fed6e0d", "object": "cross-site-source-haproxy"} 2024-07-10T13:45:04.078Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "a568bcfd-e80b-4697-87f0-f55f6641aa23", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:10.402Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "e2531da3-d74d-422d-81cd-693a19cf3799", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:15.728Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "40a6cafd-f987-4fc4-b90f-59360adb4393", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:21.142Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "48d54535-6511-4181-b36d-74f09a61259b", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:26.457Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "880def2e-2ef7-446f-bb54-2c5361d94025", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:31.858Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "d79e120e-ff8c-4d15-8050-8541439825f7", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:37.207Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "6e8a7d55-ae47-4927-80e4-18d4cad66386", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:42.625Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "e212a43e-069e-4779-8a04-cdf915c999e1", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:45.490Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "18b3a399-13ed-491d-ae07-fa28eb48d0f3", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.119.171.30:3306: connect: connection refused"} 2024-07-10T13:45:47.612Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "d3615c4e-1846-451f-82f5-3c90c285b53b", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.119.171.30:3306: connect: connection refused"} 2024-07-10T13:45:47.960Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "58613d97-c20e-4dd6-94fa-4bc19d347582", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:50.257Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "8f763e4f-9f89-483c-9a59-fe7c75e1fca9", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.119.171.30:3306: connect: connection refused"} 2024-07-10T13:45:53.316Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "69e1e770-d698-4714-b10f-146db48e8b01", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:58.663Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "3e660383-4a8a-4f29-8009-72aa8b806e13", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:04.064Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "b9dff766-be56-4d3e-9fa8-3bb719562f34", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:10.556Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "bcf051e4-c4e0-426f-b1e4-63cd0424de8e", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:15.923Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "889aa631-5b63-435f-b08e-33f556ce2111", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:21.242Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "d07df896-7977-4a21-b06f-3feb04edb8bb", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:26.740Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "398d14f2-0248-4a3f-ba2c-2ae6d2e6fa4c", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:32.149Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "e826ce46-4037-4366-b423-d5dbbc1e9672", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:37.538Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "2e73d22f-1b65-4aab-b322-34f884b24aba", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:42.898Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "856e3404-28a8-41f0-bfd8-ee2ee29d73e3", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:48.238Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "fdab5c44-a764-4723-b8a8-4f29f8110cc5", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:53.533Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "fbfd5563-9d44-4dc6-9f53-11e643b43fd5", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:58.920Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "4032aee7-c5ab-4987-bf99-58e1a25cd5eb", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:04.727Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "a06be05b-1a3e-4b7d-abfb-89ebdfdef016", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:10.519Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "aee6598e-85b9-48e2-926f-f8674c67105f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:15.915Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "60913204-b970-46d2-bf1e-38535cbf3d86", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:21.342Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "5cb04fb0-a30b-4f26-a32f-b7dd1bdaecf7", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:26.645Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "680d7ac1-a542-415a-bb9c-f20e2e7dc37d", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:32.006Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "5f406493-8060-4ae8-8737-c7bbbc16065f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:37.403Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "5e3fb83c-848a-42e6-a379-2a1c22a4554c", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:42.752Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "98e93a0f-8ca2-4c58-9fe0-9c8c6c37c398", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:48.147Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "69bbb0e6-6321-48ac-9c66-df39c5d4094e", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:53.501Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "e779123a-c36b-4e6e-a233-d1fc16e8d4ca", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:58.918Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "c092310a-07d3-48c9-87bb-ee6ab1a7b312", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:48:04.837Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "4cf29a70-fea0-4118-b6ef-6b2ffa29450e", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:48:05.738Z INFO You can view xtrabackup log: 2024-07-10T13:48:32.513Z INFO Remove outdated replication channel {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "channel": "source_to_replica"} 2024-07-10T13:48:32.517Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "channel": "source_to_replica", "host": "10.119.165.152"} 2024-07-10T13:48:32.518Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "channel": "source_to_replica", "host": "10.119.168.231"} 2024-07-10T13:48:32.519Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "channel": "source_to_replica", "host": "10.119.168.71"} 2024-07-10T13:48:32.528Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "pod": "cross-site-replica-pxc-1"} 2024-07-10T13:48:32.537Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "pod": "cross-site-replica-pxc-0"} 2024-07-10T13:48:32.548Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "pod": "cross-site-replica-pxc-2"} 2024-07-10T13:48:39.516Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "5c5307f6-aa03-4170-8dfe-857eeac0b71b", "pod": "cross-site-source-pxc-2"} 2024-07-10T13:48:39.538Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "5c5307f6-aa03-4170-8dfe-857eeac0b71b", "pod": "cross-site-source-pxc-0"} 2024-07-10T13:48:39.560Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "5c5307f6-aa03-4170-8dfe-857eeac0b71b", "pod": "cross-site-source-pxc-1"} 2024-07-10T13:48:39.599Z INFO Replication pod has changed {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "5c5307f6-aa03-4170-8dfe-857eeac0b71b", "new replication pod": "cross-site-source-pxc-0"} {"controller": "pxcrestore-controller", "namespace": "cross-site-21497", "name": "backup-minio", "reconcileID": "e62b38fd-b74d-432f-9adf-312cf9946b09"} {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-16033", "name": "backup-minio", "reconcileID": "3ef59fcb-535b-44a1-b0ce-03ca2d47ee32"} If everything is fine, you can cleanup the job: $ kubectl delete pxc-restore/backup-minio $ kubectl logs job/restore-job-backup-minio-cross-site-replica $ kubectl logs job/restore-job-backup-minio-cross-site-source + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl patch pxc -n cross-site-21497 cross-site-source --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + kubectl patch pxc -n cross-site-replica-16033 cross-site-replica --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.5Z1zUAmU2M ++ mktemp + local LAST_ERR=/tmp/tmp.HMa90QIEUg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5Z1zUAmU2M perconaxtradbcluster.pxc.percona.com "cross-site-source" deleted perconaxtradbcluster.pxc.percona.com "cross-site-replica" deleted + cat /tmp/tmp.HMa90QIEUg + rm /tmp/tmp.5Z1zUAmU2M /tmp/tmp.HMa90QIEUg + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.x4yvJBMegW ++ mktemp + local LAST_ERR=/tmp/tmp.8lFIsnvpIC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.x4yvJBMegW perconaxtradbclusterbackup.pxc.percona.com "backup-minio-source" deleted perconaxtradbclusterbackup.pxc.percona.com "backup-minio-replica" deleted + cat /tmp/tmp.8lFIsnvpIC + rm /tmp/tmp.x4yvJBMegW /tmp/tmp.8lFIsnvpIC + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.xNubRJmhCh ++ mktemp + local LAST_ERR=/tmp/tmp.wuTiLRF6lH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xNubRJmhCh perconaxtradbclusterrestore.pxc.percona.com "backup-minio" deleted perconaxtradbclusterrestore.pxc.percona.com "backup-minio" deleted + cat /tmp/tmp.wuTiLRF6lH + rm /tmp/tmp.xNubRJmhCh /tmp/tmp.wuTiLRF6lH + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.V1Y2QGD9oF ++ mktemp + local LAST_ERR=/tmp/tmp.aSYE04QMPW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.V1Y2QGD9oF validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.aSYE04QMPW + rm /tmp/tmp.V1Y2QGD9oF /tmp/tmp.aSYE04QMPW + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + rm -rf /tmp/tmp.Oz5Ph9gkPr + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator + destroy cross-site-replica-16033 true ++ mktemp + local namespace=cross-site-replica-16033 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-21497 + local LAST_OUT=/tmp/tmp.jjJvt1yPIO ++ mktemp + local LAST_ERR=/tmp/tmp.uDl5BEVjkw + local exit_status=0 + grep -v 'get backup status: Job.batch' + tee /tmp/tmp.Oz5Ph9gkPr/operator.log ++ mktemp + sort -u + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v level=info tee: /tmp/tmp.Oz5Ph9gkPr/operator.log: No such file or directory ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + local LAST_OUT=/tmp/tmp.sCn78UHD19 + grep -v 'the object has been modified' +++ grep -c percona-xtradb-cluster-operator ++ mktemp +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator + local LAST_ERR=/tmp/tmp.COHp8GpXw9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace cross-site-21497 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.PYu9wVhcDm +++ mktemp ++ local LAST_ERR=/tmp/tmp.jB9kjYGIUD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PYu9wVhcDm ++ cat /tmp/tmp.jB9kjYGIUD ++ rm /tmp/tmp.PYu9wVhcDm /tmp/tmp.jB9kjYGIUD ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-675fc7d9f5-jjk9g ++ mktemp + local LAST_OUT=/tmp/tmp.fEsOPselGv ++ mktemp + local LAST_ERR=/tmp/tmp.VgftyU5PnJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-675fc7d9f5-jjk9g + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fEsOPselGv + cat /tmp/tmp.VgftyU5PnJ + rm /tmp/tmp.fEsOPselGv /tmp/tmp.VgftyU5PnJ + return 0 2024-07-10T13:19:27.836Z INFO setup Manager starting up {"gitCommit": "51af051703ea68e6367821ec3068826842ab5c9d", "gitBranch": "PR-1743-51af0517", "buildTime": "2024-07-10T11:38:48Z", "goVersion": "go1.22.5", "os": "linux", "arch": "amd64"} 2024-07-10T13:19:27.836Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.26.15-gke.1469001"} 2024-07-10T13:19:27.837Z INFO setup Registering Components. 2024-07-10T13:19:31.113Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2024-07-10T13:19:31.116Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2024-07-10T13:19:31.116Z INFO controller-runtime.metrics Starting metrics server 2024-07-10T13:19:31.116Z INFO controller-runtime.webhook Starting webhook server 2024-07-10T13:19:31.116Z INFO setup Starting the Cmd. 2024-07-10T13:19:31.116Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2024-07-10T13:19:31.117Z INFO controller-runtime.certwatcher Starting certificate watcher 2024-07-10T13:19:31.117Z INFO controller-runtime.certwatcher Updated current TLS certificate 2024-07-10T13:19:31.117Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2024-07-10T13:19:31.218Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2024-07-10T13:19:31.233Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2024-07-10T13:19:31.234Z DEBUG events percona-xtradb-cluster-operator-675fc7d9f5-jjk9g_c7d207b8-c6f2-489d-a0b8-6d0b27ac5b96 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"0c84f1d1-331c-447b-947d-d4326fea30e6","apiVersion":"coordination.k8s.io/v1","resourceVersion":"48182"}, "reason": "LeaderElection"} 2024-07-10T13:19:31.234Z INFO Starting Controller {"controller": "pxc-controller"} 2024-07-10T13:19:31.234Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2024-07-10T13:19:31.234Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"} 2024-07-10T13:19:31.234Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2024-07-10T13:19:31.235Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2024-07-10T13:19:31.235Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2024-07-10T13:19:31.446Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2024-07-10T13:19:31.446Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2024-07-10T13:19:31.446Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2024-07-10T13:21:47.899Z INFO Set CR version {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "version": "1.15.0"} 2024-07-10T13:22:02.230Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-pxc"} 2024-07-10T13:22:02.354Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-pxc"} 2024-07-10T13:22:02.438Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-pxc-unready"} 2024-07-10T13:22:02.552Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-pxc-0"} 2024-07-10T13:22:02.759Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-pxc-1"} 2024-07-10T13:22:02.925Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-pxc-2"} 2024-07-10T13:22:02.989Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-haproxy"} 2024-07-10T13:22:03.028Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-haproxy"} 2024-07-10T13:22:03.064Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "0bebf063-52c5-464e-8c8b-7c75257abe0c", "object": "cross-site-source-haproxy-replicas"} 2024-07-10T13:22:03.156Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 5b91f771-be74-47e4-aaa7-7f0007fd41d1 2024-07-10T13:22:03.356Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "044f02d3-2db3-4dd2-aaff-f26b8b178e48", "object": "cross-site-source-pxc"} 2024-07-10T13:22:03.393Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "044f02d3-2db3-4dd2-aaff-f26b8b178e48", "object": "cross-site-source-haproxy"} 2024-07-10T13:23:24.954Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a", "user": "operator"} 2024-07-10T13:23:24.997Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a", "user": "monitor"} 2024-07-10T13:23:25.094Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a"} 2024-07-10T13:23:25.130Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a"} 2024-07-10T13:23:25.172Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a", "user": "xtrabackup"} 2024-07-10T13:23:25.250Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a"} 2024-07-10T13:23:25.317Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ad19c5e0-2583-4a78-99b7-3a36caee763a", "user": "replication"} 2024-07-10T13:25:46.297Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ee3c7958-b364-45eb-bfd2-ef70b1773755", "user": "root"} 2024-07-10T13:25:46.564Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "ee3c7958-b364-45eb-bfd2-ef70b1773755", "new version": "8.0.36-28.1"} 2024-07-10T13:28:14.411Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "17dbfb82-8ecd-45e8-8e1b-21a9b5e1f239", "user": "replication"} 2024-07-10T13:28:14.447Z INFO Password updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "17dbfb82-8ecd-45e8-8e1b-21a9b5e1f239", "user": "replication"} 2024-07-10T13:28:14.455Z INFO MySQL init secret created {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "17dbfb82-8ecd-45e8-8e1b-21a9b5e1f239", "secret": "cross-site-source-mysql-init", "user": "replication"} 2024-07-10T13:28:14.467Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "17dbfb82-8ecd-45e8-8e1b-21a9b5e1f239", "user": "replication"} 2024-07-10T13:28:14.495Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "17dbfb82-8ecd-45e8-8e1b-21a9b5e1f239", "user": "replication"} 2024-07-10T13:28:52.038Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "cross-site-21497", "name": "backup-minio-source", "reconcileID": "0ee0c548-206c-4d42-9d62-419433b2c0d0", "Namespace": "cross-site-21497", "Name": "xb-backup-minio-source"} 2024-07-10T13:30:53.681Z INFO Set CR version {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "version": "1.15.0"} 2024-07-10T13:31:01.424Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-pxc"} 2024-07-10T13:31:01.478Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-haproxy"} 2024-07-10T13:31:01.509Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-pxc"} 2024-07-10T13:31:01.578Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-pxc-unready"} 2024-07-10T13:31:01.751Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-pxc-0"} 2024-07-10T13:31:01.984Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-pxc-1"} 2024-07-10T13:31:02.071Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-pxc-2"} 2024-07-10T13:31:02.150Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-haproxy"} 2024-07-10T13:31:02.188Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-haproxy"} 2024-07-10T13:31:02.228Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f1265cde-5468-4f13-a962-2daf7133fb7e", "object": "cross-site-replica-haproxy-replicas"} 2024-07-10T13:31:02.268Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: f279ae47-e06d-4fe2-a4e0-f03232971010 2024-07-10T13:31:02.727Z DEBUG Creating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "261dc436-45a0-497f-85c4-a1fa51eea25f", "object": "cross-site-replica-pxc"} 2024-07-10T13:32:18.854Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0", "user": "operator"} 2024-07-10T13:32:18.895Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0", "user": "monitor"} 2024-07-10T13:32:18.987Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0"} 2024-07-10T13:32:19.023Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0"} 2024-07-10T13:32:19.059Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0", "user": "xtrabackup"} 2024-07-10T13:32:19.349Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0"} 2024-07-10T13:32:19.601Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ab09d47a-81ca-437a-be30-de32d3c534d0", "user": "replication"} 2024-07-10T13:34:53.296Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f12c590b-e439-45ca-930a-834047119a57", "user": "root"} 2024-07-10T13:34:53.563Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "f12c590b-e439-45ca-930a-834047119a57", "new version": "8.0.36-28.1"} 2024-07-10T13:37:07.388Z INFO backup restore request {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-16033", "name": "backup-minio", "reconcileID": "3ef59fcb-535b-44a1-b0ce-03ca2d47ee32"} 2024-07-10T13:37:07.438Z INFO stopping cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-16033", "name": "backup-minio", "reconcileID": "3ef59fcb-535b-44a1-b0ce-03ca2d47ee32", "cluster": "cross-site-replica"} 2024-07-10T13:37:07.568Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "14e470ec-5823-499b-92d7-f3a14c597881", "object": "cross-site-replica-pxc"} 2024-07-10T13:37:07.674Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "14e470ec-5823-499b-92d7-f3a14c597881", "object": "cross-site-replica-haproxy"} 2024-07-10T13:37:53.577Z INFO starting restore {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-16033", "name": "backup-minio", "reconcileID": "3ef59fcb-535b-44a1-b0ce-03ca2d47ee32", "cluster": "cross-site-replica", "backup": ""} 2024-07-10T13:38:12.665Z INFO starting cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-16033", "name": "backup-minio", "reconcileID": "3ef59fcb-535b-44a1-b0ce-03ca2d47ee32", "cluster": "cross-site-replica"} 2024-07-10T13:38:12.740Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "74a31a7e-c5d3-4cb9-a386-801db39b5aac", "object": "cross-site-replica-pxc"} 2024-07-10T13:38:12.807Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "74a31a7e-c5d3-4cb9-a386-801db39b5aac", "object": "cross-site-replica-haproxy"} 2024-07-10T13:41:16.870Z INFO You can view xtrabackup log: 2024-07-10T13:41:47.911Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "00e9e3ee-e1ab-4853-a70c-ee70eace116f", "pod": "cross-site-replica-pxc-2"} 2024-07-10T13:41:47.930Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "00e9e3ee-e1ab-4853-a70c-ee70eace116f", "pod": "cross-site-replica-pxc-0"} 2024-07-10T13:41:47.951Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "00e9e3ee-e1ab-4853-a70c-ee70eace116f", "pod": "cross-site-replica-pxc-1"} 2024-07-10T13:41:48.013Z INFO Replication pod has changed {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "00e9e3ee-e1ab-4853-a70c-ee70eace116f", "new replication pod": "cross-site-replica-pxc-0"} 2024-07-10T13:41:49.959Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "4bb9ecfa-7d9a-4ae1-ac34-ce66f68624e4", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:41:53.946Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "8529ecd8-065d-4112-91c9-e943da37060e", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:41:59.309Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "3f6b6c30-fa3b-4d52-b552-2380e94da519", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:42:05.432Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "91e5ecb0-9a77-4527-a88e-916567946568", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:42:10.910Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "b12ef1aa-9ef1-4b7a-95fb-c5318c836035", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:42:16.257Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "1d4f177e-e8bd-4937-a71c-dd9e52f8dca8", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:42:21.569Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "d93f570a-541b-4039-a30c-625df3b093c2", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:42:26.964Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "ea02bbbe-5098-4693-90d6-d78a7d836692", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-07-10T13:42:32.185Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "dd37927e-a02d-48b2-9325-b30c8a315c60", "user": "replication"} 2024-07-10T13:42:32.234Z INFO Password updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "dd37927e-a02d-48b2-9325-b30c8a315c60", "user": "replication"} 2024-07-10T13:42:32.241Z INFO MySQL init secret created {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "dd37927e-a02d-48b2-9325-b30c8a315c60", "secret": "cross-site-replica-mysql-init", "user": "replication"} 2024-07-10T13:42:32.255Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "dd37927e-a02d-48b2-9325-b30c8a315c60", "user": "replication"} 2024-07-10T13:42:32.285Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "dd37927e-a02d-48b2-9325-b30c8a315c60", "user": "replication"} 2024-07-10T13:43:13.366Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "cross-site-replica-16033", "name": "backup-minio-replica", "reconcileID": "4a375727-d790-49f0-bd18-f662d445b22a", "Namespace": "cross-site-replica-16033", "Name": "xb-backup-minio-replica"} 2024-07-10T13:43:48.128Z INFO backup restore request {"controller": "pxcrestore-controller", "namespace": "cross-site-21497", "name": "backup-minio", "reconcileID": "e62b38fd-b74d-432f-9adf-312cf9946b09"} 2024-07-10T13:43:48.209Z INFO stopping cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-21497", "name": "backup-minio", "reconcileID": "e62b38fd-b74d-432f-9adf-312cf9946b09", "cluster": "cross-site-source"} 2024-07-10T13:43:48.415Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "261580cc-dc4e-4183-b647-52768e730334", "object": "cross-site-source-pxc"} 2024-07-10T13:43:48.515Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "261580cc-dc4e-4183-b647-52768e730334", "object": "cross-site-source-haproxy"} 2024-07-10T13:44:10.416Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "78f12912-32e7-4000-bdfb-856267406fbc", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:15.821Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "36d0e170-4930-4076-8193-0902b78b41ac", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:21.143Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "c1ac7902-5c2f-42df-93db-88c1ab068114", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:26.570Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "05f53885-47ba-477d-b686-cd0e7e5eee13", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:31.920Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "c779025d-5c6d-46a4-9a76-022eb46ab99f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:37.238Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "dcaa2b08-2d72-4f6d-a0b8-adf7eaef6c54", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:40.407Z INFO starting restore {"controller": "pxcrestore-controller", "namespace": "cross-site-21497", "name": "backup-minio", "reconcileID": "e62b38fd-b74d-432f-9adf-312cf9946b09", "cluster": "cross-site-source", "backup": ""} 2024-07-10T13:44:42.620Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "d2dc6447-6cdb-4d65-9880-5a4396eadb86", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:48.015Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "2a158491-89f5-4ffa-a67e-fdf89185cd77", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:53.332Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "e6ea218d-7843-4737-8522-8402156da593", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:44:58.728Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "c05c7052-ff96-4d07-81dd-541b0412bbf1", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:00.531Z INFO starting cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-21497", "name": "backup-minio", "reconcileID": "e62b38fd-b74d-432f-9adf-312cf9946b09", "cluster": "cross-site-source"} 2024-07-10T13:45:00.590Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "d0772ca0-6153-46b9-bac1-c2117fed6e0d", "object": "cross-site-source-pxc"} 2024-07-10T13:45:00.678Z DEBUG Updating object {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "d0772ca0-6153-46b9-bac1-c2117fed6e0d", "object": "cross-site-source-haproxy"} 2024-07-10T13:45:04.078Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "a568bcfd-e80b-4697-87f0-f55f6641aa23", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:10.402Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "e2531da3-d74d-422d-81cd-693a19cf3799", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:15.728Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "40a6cafd-f987-4fc4-b90f-59360adb4393", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:21.142Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "48d54535-6511-4181-b36d-74f09a61259b", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:26.457Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "880def2e-2ef7-446f-bb54-2c5361d94025", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:31.858Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "d79e120e-ff8c-4d15-8050-8541439825f7", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:37.207Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "6e8a7d55-ae47-4927-80e4-18d4cad66386", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:42.625Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "e212a43e-069e-4779-8a04-cdf915c999e1", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:45.490Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "18b3a399-13ed-491d-ae07-fa28eb48d0f3", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.119.171.30:3306: connect: connection refused"} 2024-07-10T13:45:47.612Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "d3615c4e-1846-451f-82f5-3c90c285b53b", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.119.171.30:3306: connect: connection refused"} 2024-07-10T13:45:47.960Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "58613d97-c20e-4dd6-94fa-4bc19d347582", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:50.257Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "8f763e4f-9f89-483c-9a59-fe7c75e1fca9", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.119.171.30:3306: connect: connection refused"} 2024-07-10T13:45:53.316Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "69e1e770-d698-4714-b10f-146db48e8b01", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:45:58.663Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "3e660383-4a8a-4f29-8009-72aa8b806e13", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:04.064Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "b9dff766-be56-4d3e-9fa8-3bb719562f34", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:10.556Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "bcf051e4-c4e0-426f-b1e4-63cd0424de8e", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:15.923Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "889aa631-5b63-435f-b08e-33f556ce2111", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:21.242Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "d07df896-7977-4a21-b06f-3feb04edb8bb", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:26.740Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "398d14f2-0248-4a3f-ba2c-2ae6d2e6fa4c", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:32.149Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "e826ce46-4037-4366-b423-d5dbbc1e9672", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:37.538Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "2e73d22f-1b65-4aab-b322-34f884b24aba", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:42.898Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "856e3404-28a8-41f0-bfd8-ee2ee29d73e3", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:48.238Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "fdab5c44-a764-4723-b8a8-4f29f8110cc5", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:53.533Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "fbfd5563-9d44-4dc6-9f53-11e643b43fd5", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:46:58.920Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "4032aee7-c5ab-4987-bf99-58e1a25cd5eb", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:04.727Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "a06be05b-1a3e-4b7d-abfb-89ebdfdef016", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:10.519Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "aee6598e-85b9-48e2-926f-f8674c67105f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:15.915Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "60913204-b970-46d2-bf1e-38535cbf3d86", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:21.342Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "5cb04fb0-a30b-4f26-a32f-b7dd1bdaecf7", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:26.645Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "680d7ac1-a542-415a-bb9c-f20e2e7dc37d", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:32.006Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "5f406493-8060-4ae8-8737-c7bbbc16065f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:37.403Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "5e3fb83c-848a-42e6-a379-2a1c22a4554c", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:42.752Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "98e93a0f-8ca2-4c58-9fe0-9c8c6c37c398", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:48.147Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "69bbb0e6-6321-48ac-9c66-df39c5d4094e", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:53.501Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "e779123a-c36b-4e6e-a233-d1fc16e8d4ca", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:47:58.918Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "c092310a-07d3-48c9-87bb-ee6ab1a7b312", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:48:04.837Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "4cf29a70-fea0-4118-b6ef-6b2ffa29450e", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.119.168.231:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.119.168.231:3306' (111)"} 2024-07-10T13:48:05.738Z INFO You can view xtrabackup log: 2024-07-10T13:48:32.513Z INFO Remove outdated replication channel {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "channel": "source_to_replica"} 2024-07-10T13:48:32.517Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "channel": "source_to_replica", "host": "10.119.165.152"} 2024-07-10T13:48:32.518Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "channel": "source_to_replica", "host": "10.119.168.231"} 2024-07-10T13:48:32.519Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "channel": "source_to_replica", "host": "10.119.168.71"} 2024-07-10T13:48:32.528Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "pod": "cross-site-replica-pxc-1"} 2024-07-10T13:48:32.537Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "pod": "cross-site-replica-pxc-0"} 2024-07-10T13:48:32.548Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "47c4c3e2-300c-4444-bbee-8c876731adb7", "pod": "cross-site-replica-pxc-2"} 2024-07-10T13:48:39.516Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "5c5307f6-aa03-4170-8dfe-857eeac0b71b", "pod": "cross-site-source-pxc-2"} 2024-07-10T13:48:39.538Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "5c5307f6-aa03-4170-8dfe-857eeac0b71b", "pod": "cross-site-source-pxc-0"} 2024-07-10T13:48:39.560Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "5c5307f6-aa03-4170-8dfe-857eeac0b71b", "pod": "cross-site-source-pxc-1"} 2024-07-10T13:48:39.599Z INFO Replication pod has changed {"controller": "pxc-controller", "namespace": "cross-site-21497", "name": "cross-site-source", "reconcileID": "5c5307f6-aa03-4170-8dfe-857eeac0b71b", "new replication pod": "cross-site-source-pxc-0"} 2024-07-10T13:49:34.937Z ERROR Update status {"controller": "pxc-controller", "namespace": "cross-site-replica-16033", "name": "cross-site-replica", "reconcileID": "c1026ebe-7899-4b20-bd24-84a791384300", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"cross-site-replica\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"cross-site-replica\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:203\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:446\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} {"controller": "pxcrestore-controller", "namespace": "cross-site-21497", "name": "backup-minio", "reconcileID": "e62b38fd-b74d-432f-9adf-312cf9946b09"} {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-16033", "name": "backup-minio", "reconcileID": "3ef59fcb-535b-44a1-b0ce-03ca2d47ee32"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:205 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:446 If everything is fine, you can cleanup the job: $ kubectl delete pxc-restore/backup-minio $ kubectl logs job/restore-job-backup-minio-cross-site-replica $ kubectl logs job/restore-job-backup-minio-cross-site-source sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 + : + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.MsRjgoYPTs ++ mktemp + local LAST_ERR=/tmp/tmp.rqqBPI1TgF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MsRjgoYPTs No resources found + cat /tmp/tmp.rqqBPI1TgF + rm /tmp/tmp.MsRjgoYPTs /tmp/tmp.rqqBPI1TgF + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.4VwKM4Thp0 ++ mktemp + local LAST_ERR=/tmp/tmp.cvmkmJrL8K + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4VwKM4Thp0 No resources found + cat /tmp/tmp.cvmkmJrL8K + rm /tmp/tmp.4VwKM4Thp0 /tmp/tmp.cvmkmJrL8K + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.MAQdU7o4A8 ++ mktemp + local LAST_ERR=/tmp/tmp.hzI9Y9lm1v + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MAQdU7o4A8 No resources found + cat /tmp/tmp.hzI9Y9lm1v + rm /tmp/tmp.MAQdU7o4A8 /tmp/tmp.hzI9Y9lm1v + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.GNNFvxRRCc ++ mktemp + local LAST_ERR=/tmp/tmp.CWbwWw7uhB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.GNNFvxRRCc + cat /tmp/tmp.CWbwWw7uhB Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "percona-xtradbcluster-webhook" not found + rm /tmp/tmp.GNNFvxRRCc /tmp/tmp.CWbwWw7uhB + return 1 + : + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jjJvt1yPIO namespace "pxc-operator" force deleted + cat /tmp/tmp.uDl5BEVjkw Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.jjJvt1yPIO /tmp/tmp.uDl5BEVjkw + return 0 + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-replica-16033 + rm -rf /tmp/tmp.Oz5Ph9gkPr + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ mktemp + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.GSSOeA1xCq + local LAST_OUT=/tmp/tmp.PZEJe07XGW ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.s5unL9lG8g + local exit_status=0 + local LAST_ERR=/tmp/tmp.Jp3LL9TqSc + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace cross-site-replica-16033 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.PZEJe07XGW + cat /tmp/tmp.s5unL9lG8g Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. Error from server (NotFound): namespaces "pxc-operator" not found + rm /tmp/tmp.PZEJe07XGW /tmp/tmp.s5unL9lG8g + return 1