Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/logs/cross-site-8-0.log WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + main + local source_cluster=cross-site-source + local replica_cluster=cross-site-replica + desc 'Create source cluster' + set +o xtrace ----------------------------------------------------------------------------------- Create source cluster ----------------------------------------------------------------------------------- + create_infra cross-site-27746 + local ns=cross-site-27746 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n cross-site-19758 cross-site-source --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + kubectl patch pxc -n cross-site-replica-3390 cross-site-replica --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.vFy9xzQZnr ++ mktemp + local LAST_ERR=/tmp/tmp.aF5NpkQS5I + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vFy9xzQZnr perconaxtradbcluster.pxc.percona.com "cross-site-source" deleted perconaxtradbcluster.pxc.percona.com "cross-site-replica" deleted + cat /tmp/tmp.aF5NpkQS5I + rm /tmp/tmp.vFy9xzQZnr /tmp/tmp.aF5NpkQS5I + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.ltIb4byDMt ++ mktemp + local LAST_ERR=/tmp/tmp.L0o9WxzSHo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ltIb4byDMt perconaxtradbclusterbackup.pxc.percona.com "backup-minio-source" deleted perconaxtradbclusterbackup.pxc.percona.com "backup-minio-replica" deleted + cat /tmp/tmp.L0o9WxzSHo + rm /tmp/tmp.ltIb4byDMt /tmp/tmp.L0o9WxzSHo + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.zEiUEe8upn ++ mktemp + local LAST_ERR=/tmp/tmp.7Ic5KSuvnR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zEiUEe8upn perconaxtradbclusterrestore.pxc.percona.com "backup-minio" deleted perconaxtradbclusterrestore.pxc.percona.com "backup-minio" deleted + cat /tmp/tmp.7Ic5KSuvnR + rm /tmp/tmp.zEiUEe8upn /tmp/tmp.7Ic5KSuvnR + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + awk '{print$1}' + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.HgeyqKSBeJ + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.ir0ilovQDz + local exit_status=0 + local LAST_OUT=/tmp/tmp.LTUt5otvAf ++ seq 0 2 ++ mktemp + local LAST_ERR=/tmp/tmp.8Eli4kj8sS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LTUt5otvAf + cat /tmp/tmp.8Eli4kj8sS + rm /tmp/tmp.LTUt5otvAf /tmp/tmp.8Eli4kj8sS + return 0 namespace "cross-site-19758" deleted namespace "cross-site-replica-3390" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HgeyqKSBeJ namespace "pxc-operator" deleted + cat /tmp/tmp.ir0ilovQDz + rm /tmp/tmp.HgeyqKSBeJ /tmp/tmp.ir0ilovQDz + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'namespace/pxc-operator - ' namespace/pxc-operator - + set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ecmX8AL4Ya ++ mktemp + local LAST_ERR=/tmp/tmp.xB2JnMIVie + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ecmX8AL4Ya namespace/pxc-operator created + cat /tmp/tmp.xB2JnMIVie + rm /tmp/tmp.ecmX8AL4Ya /tmp/tmp.xB2JnMIVie + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.lqFpVTGOg5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7M4wf29k68 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lqFpVTGOg5 ++ cat /tmp/tmp.7M4wf29k68 ++ rm /tmp/tmp.lqFpVTGOg5 /tmp/tmp.7M4wf29k68 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.S6hx4FRaQF ++ mktemp + local LAST_ERR=/tmp/tmp.324gAkBoaJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.S6hx4FRaQF Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3" modified. + cat /tmp/tmp.324gAkBoaJ + rm /tmp/tmp.S6hx4FRaQF /tmp/tmp.324gAkBoaJ + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.4tY7QQBNGQ ++ mktemp + local LAST_ERR=/tmp/tmp.VJqCtmnMgZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4tY7QQBNGQ customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.VJqCtmnMgZ + rm /tmp/tmp.4tY7QQBNGQ /tmp/tmp.VJqCtmnMgZ + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/deploy/cw-rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.MjaecDt6Db ++ mktemp + local LAST_ERR=/tmp/tmp.JlnznJ6HNY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MjaecDt6Db clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.JlnznJ6HNY + rm /tmp/tmp.MjaecDt6Db /tmp/tmp.JlnznJ6HNY + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1740-0a840b68^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.zp2ZCCfUTS ++ mktemp + local LAST_ERR=/tmp/tmp.N7pAvXiOSR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zp2ZCCfUTS deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.N7pAvXiOSR + rm /tmp/tmp.zp2ZCCfUTS /tmp/tmp.N7pAvXiOSR + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.lWaoPWE1pK ++ mktemp + local LAST_ERR=/tmp/tmp.vvBNDexrbJ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lWaoPWE1pK pod/percona-xtradb-cluster-operator-6849457d9-c7dd7 condition met + cat /tmp/tmp.vvBNDexrbJ + rm /tmp/tmp.lWaoPWE1pK /tmp/tmp.vvBNDexrbJ + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.SFVzhb6VNZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.bZ17IwrZ17 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SFVzhb6VNZ ++ cat /tmp/tmp.bZ17IwrZ17 ++ rm /tmp/tmp.SFVzhb6VNZ /tmp/tmp.bZ17IwrZ17 ++ return 0 + wait_pod percona-xtradb-cluster-operator-6849457d9-c7dd7 480 pxc-operator + local pod=percona-xtradb-cluster-operator-6849457d9-c7dd7 + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-6849457d9-c7dd7 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-6849457d9-c7dd7 condition met percona-xtradb-cluster-operator-6849457d9-c7dd7.Ok + sleep 3 + create_namespace cross-site-27746 + local namespace=cross-site-27746 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-27746' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-27746 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-27746 + xargs kubectl delete ns + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.v9afKeyHoD ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.67Pfa9b1Po + local exit_status=0 + local LAST_OUT=/tmp/tmp.CNjKFye5jn ++ seq 0 2 ++ mktemp + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.27tQKppYLe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-27746 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.v9afKeyHoD + cat /tmp/tmp.67Pfa9b1Po + rm /tmp/tmp.v9afKeyHoD /tmp/tmp.67Pfa9b1Po + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-27746 Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-27746 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.CNjKFye5jn + cat /tmp/tmp.27tQKppYLe Error from server (NotFound): namespaces "cross-site-27746" not found + rm /tmp/tmp.CNjKFye5jn /tmp/tmp.27tQKppYLe + return 1 + : + wait_for_delete namespace/cross-site-27746 + local res=namespace/cross-site-27746 + echo -n 'namespace/cross-site-27746 - ' namespace/cross-site-27746 - + set +o xtrace Error from server (NotFound): namespaces "cross-site-27746" not found + desc 'create namespace cross-site-27746' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-27746 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-27746 ++ mktemp + local LAST_OUT=/tmp/tmp.mVnsTYczks ++ mktemp + local LAST_ERR=/tmp/tmp.RLEqfhnoha + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cross-site-27746 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mVnsTYczks namespace/cross-site-27746 created + cat /tmp/tmp.RLEqfhnoha + rm /tmp/tmp.mVnsTYczks /tmp/tmp.RLEqfhnoha + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.wh5wv8jn7e +++ mktemp ++ local LAST_ERR=/tmp/tmp.2NwFpCAf3h ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wh5wv8jn7e ++ cat /tmp/tmp.2NwFpCAf3h ++ rm /tmp/tmp.wh5wv8jn7e /tmp/tmp.2NwFpCAf3h ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3 --namespace=cross-site-27746 ++ mktemp + local LAST_OUT=/tmp/tmp.p3tlAYwR5L ++ mktemp + local LAST_ERR=/tmp/tmp.Gf8UdXaKlW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3 --namespace=cross-site-27746 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.p3tlAYwR5L Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3" modified. + cat /tmp/tmp.Gf8UdXaKlW + rm /tmp/tmp.p3tlAYwR5L /tmp/tmp.Gf8UdXaKlW + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ZQXzDYo3CS ++ mktemp + local LAST_ERR=/tmp/tmp.F6yEUvmAzh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZQXzDYo3CS secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.F6yEUvmAzh + rm /tmp/tmp.ZQXzDYo3CS /tmp/tmp.F6yEUvmAzh + return 0 + start_minio + deploy_helm cross-site-27746 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "chaos-mesh" chart repository ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Fri Jun 28 07:48:21 2024 NAMESPACE: cross-site-27746 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.cross-site-27746.svc.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace cross-site-27746 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace cross-site-27746 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace cross-site-27746 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace cross-site-27746 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XwELKojnJc +++ mktemp ++ local LAST_ERR=/tmp/tmp.iyZKVXwf61 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XwELKojnJc ++ cat /tmp/tmp.iyZKVXwf61 ++ rm /tmp/tmp.XwELKojnJc /tmp/tmp.iyZKVXwf61 ++ return 0 + MINIO_POD=minio-service-76ffcfd45-6dmdt + wait_pod minio-service-76ffcfd45-6dmdt + local pod=minio-service-76ffcfd45-6dmdt + local max_retry=480 + local ns= ++ echo minio-service-76ffcfd45-6dmdt ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minio-service-76ffcfd45-6dmdt condition met minio-service-76ffcfd45-6dmdt.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.03QfVC21Er ++ mktemp + local LAST_ERR=/tmp/tmp.v0XTtOsg1O + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.03QfVC21Er make_bucket: operator-testing pod "aws-cli" deleted + cat /tmp/tmp.v0XTtOsg1O If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_cross-site-27746 + rm /tmp/tmp.03QfVC21Er /tmp/tmp.v0XTtOsg1O + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.XYGXkdLSvE ++ mktemp + local LAST_ERR=/tmp/tmp.bnpmuNumBO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XYGXkdLSvE secret/minio-secret unchanged secret/aws-s3-secret unchanged secret/gcp-cs-secret unchanged secret/azure-secret unchanged + cat /tmp/tmp.bnpmuNumBO + rm /tmp/tmp.XYGXkdLSvE /tmp/tmp.bnpmuNumBO + return 0 + spinup_pxc cross-site-source /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/cross-site-source.yml + local cluster=cross-site-source + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/cross-site-source.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.R7qesO0CTT ++ mktemp + local LAST_ERR=/tmp/tmp.L9HGnIXBIK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.R7qesO0CTT secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.L9HGnIXBIK + rm /tmp/tmp.R7qesO0CTT /tmp/tmp.L9HGnIXBIK + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.Q4mQ3hBCsY + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-27746~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1740-0a840b68#' + local LAST_ERR=/tmp/tmp.KVII7h6Fbi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Q4mQ3hBCsY deployment.apps/pxc-client created + cat /tmp/tmp.KVII7h6Fbi + rm /tmp/tmp.Q4mQ3hBCsY /tmp/tmp.KVII7h6Fbi + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/cross-site-source.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/cross-site-source.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/cross-site-source.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.JNjxW1MXXu ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_ERR=/tmp/tmp.ojy84jHkzc + local exit_status=0 + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1740-0a840b68#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ seq 0 2 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-27746~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JNjxW1MXXu perconaxtradbcluster.pxc.percona.com/cross-site-source created + cat /tmp/tmp.ojy84jHkzc + rm /tmp/tmp.JNjxW1MXXu /tmp/tmp.ojy84jHkzc + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy cross-site-source ++ local target_cluster=cross-site-source +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6ehl5NUjOy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9CDodGcOTv +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.6ehl5NUjOy +++ cat /tmp/tmp.9CDodGcOTv +++ rm /tmp/tmp.6ehl5NUjOy /tmp/tmp.9CDodGcOTv +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo cross-site-source-haproxy ++ return + local proxy=cross-site-source-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-27746 ++ mktemp + local LAST_OUT=/tmp/tmp.uQhLmxgGI2 ++ mktemp + local LAST_ERR=/tmp/tmp.FOBsoemRmN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-27746 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-27746 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-27746 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.uQhLmxgGI2 + cat /tmp/tmp.FOBsoemRmN error: no matching resources found + rm /tmp/tmp.uQhLmxgGI2 /tmp/tmp.FOBsoemRmN + return 1 + true + wait_for_running cross-site-source-haproxy 1 + local name=cross-site-source-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-haproxy-0 480 + local pod=cross-site-source-haproxy-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo cross-site-source-haproxy-0 + local container= + set +o xtrace pod/cross-site-source-haproxy-0 condition met cross-site-source-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running cross-site-source-pxc 3 + local name=cross-site-source-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-0 480 + local pod=cross-site-source-pxc-0 + local max_retry=480 + local ns= ++ echo cross-site-source-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-source-pxc-0 condition met cross-site-source-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-1 480 + local pod=cross-site-source-pxc-1 + local max_retry=480 + local ns= ++ echo cross-site-source-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-source-pxc-1 condition met cross-site-source-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-2 480 + local pod=cross-site-source-pxc-2 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo cross-site-source-pxc-2 + local container=pxc + set +o xtrace pod/cross-site-source-pxc-2 condition met cross-site-source-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h cross-site-source-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h cross-site-source-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tUxkHsUxIn +++ mktemp ++ local LAST_ERR=/tmp/tmp.ma611wGvR1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tUxkHsUxIn ++ cat /tmp/tmp.ma611wGvR1 ++ rm /tmp/tmp.tUxkHsUxIn /tmp/tmp.ma611wGvR1 ++ return 0 + client_pod=pxc-client-6644d8898f-mzxrn + wait_pod pxc-client-6644d8898f-mzxrn + local pod=pxc-client-6644d8898f-mzxrn + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-mzxrn ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h cross-site-source-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h cross-site-source-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MWzGra0BX2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.MnIyPUtjMs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MWzGra0BX2 ++ cat /tmp/tmp.MnIyPUtjMs ++ rm /tmp/tmp.MWzGra0BX2 /tmp/tmp.MnIyPUtjMs ++ return 0 + client_pod=pxc-client-6644d8898f-mzxrn + wait_pod pxc-client-6644d8898f-mzxrn + local pod=pxc-client-6644d8898f-mzxrn + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-mzxrn ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uuIl5Tpi82 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7SYjISsfxo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uuIl5Tpi82 ++ cat /tmp/tmp.7SYjISsfxo ++ rm /tmp/tmp.uuIl5Tpi82 /tmp/tmp.7SYjISsfxo ++ return 0 + client_pod=pxc-client-6644d8898f-mzxrn + wait_pod pxc-client-6644d8898f-mzxrn + local pod=pxc-client-6644d8898f-mzxrn + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-mzxrn ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Su6U2NCsGL/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.Su6U2NCsGL/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3urgcCclW9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.thU5vAi7bt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3urgcCclW9 ++ cat /tmp/tmp.thU5vAi7bt ++ rm /tmp/tmp.3urgcCclW9 /tmp/tmp.thU5vAi7bt ++ return 0 + client_pod=pxc-client-6644d8898f-mzxrn + wait_pod pxc-client-6644d8898f-mzxrn + local pod=pxc-client-6644d8898f-mzxrn + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-mzxrn ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Su6U2NCsGL/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.Su6U2NCsGL/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TJ9ZViGF3V +++ mktemp ++ local LAST_ERR=/tmp/tmp.FkRKh3dA9s ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TJ9ZViGF3V ++ cat /tmp/tmp.FkRKh3dA9s ++ rm /tmp/tmp.TJ9ZViGF3V /tmp/tmp.FkRKh3dA9s ++ return 0 + client_pod=pxc-client-6644d8898f-mzxrn + wait_pod pxc-client-6644d8898f-mzxrn + local pod=pxc-client-6644d8898f-mzxrn + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-mzxrn + local container= + set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Su6U2NCsGL/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.Su6U2NCsGL/select-1.sql ++ is_keyring_plugin_in_use cross-site-source ++ local cluster=cross-site-source ++ kubectl_bin exec -it cross-site-source-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5reBu2KTCt +++ mktemp ++ local LAST_ERR=/tmp/tmp.O9PLLmmgh5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it cross-site-source-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5reBu2KTCt ++ cat /tmp/tmp.O9PLLmmgh5 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.5reBu2KTCt /tmp/tmp.O9PLLmmgh5 ++ return 0 + '[' '' ']' + sleep 60 + desc 'get main cluster services endpoints' + set +o xtrace ----------------------------------------------------------------------------------- get main cluster services endpoints ----------------------------------------------------------------------------------- ++ get_service_ip cross-site-source-pxc-0 ++ local service=cross-site-source-pxc-0 ++ grep -q NotFound ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gLtdYxKZ8g ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RTpomkox8d +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.gLtdYxKZ8g +++ cat /tmp/tmp.RTpomkox8d +++ rm /tmp/tmp.gLtdYxKZ8g /tmp/tmp.RTpomkox8d +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eJK0NpXcGw +++ mktemp ++ local LAST_ERR=/tmp/tmp.3GG5psMECf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eJK0NpXcGw ++ cat /tmp/tmp.3GG5psMECf ++ rm /tmp/tmp.eJK0NpXcGw /tmp/tmp.3GG5psMECf ++ return 0 ++ return + source_endpoint0=10.61.125.225 ++ get_service_ip cross-site-source-pxc-1 ++ local service=cross-site-source-pxc-1 ++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CKV31UotwB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ysasG9A9VC +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.CKV31UotwB +++ cat /tmp/tmp.ysasG9A9VC +++ rm /tmp/tmp.CKV31UotwB /tmp/tmp.ysasG9A9VC +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IP4pZMoeqK +++ mktemp ++ local LAST_ERR=/tmp/tmp.FM9tHlkz5d ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IP4pZMoeqK ++ cat /tmp/tmp.FM9tHlkz5d ++ rm /tmp/tmp.IP4pZMoeqK /tmp/tmp.FM9tHlkz5d ++ return 0 ++ return + source_endpoint1=10.61.122.184 ++ get_service_ip cross-site-source-pxc-2 ++ local service=cross-site-source-pxc-2 ++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GIxKoPfj1B ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YjlVhKgV31 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.GIxKoPfj1B +++ cat /tmp/tmp.YjlVhKgV31 +++ rm /tmp/tmp.GIxKoPfj1B /tmp/tmp.YjlVhKgV31 +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kNU3vIunx1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.5aQfaUA6w7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kNU3vIunx1 ++ cat /tmp/tmp.5aQfaUA6w7 ++ rm /tmp/tmp.kNU3vIunx1 /tmp/tmp.5aQfaUA6w7 ++ return 0 ++ return + source_endpoint2=10.61.115.122 ++ run_mysql 'SELECT @@hostname hostname;' '-h cross-site-source-haproxy -p33062 -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h cross-site-source-haproxy -p33062 -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.nlinjSv8W3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XhVcmm3afh +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.nlinjSv8W3 +++ cat /tmp/tmp.XhVcmm3afh +++ rm /tmp/tmp.nlinjSv8W3 /tmp/tmp.XhVcmm3afh +++ return 0 ++ client_pod=pxc-client-6644d8898f-mzxrn ++ wait_pod pxc-client-6644d8898f-mzxrn ++ local pod=pxc-client-6644d8898f-mzxrn ++ local max_retry=480 ++ local ns= +++ echo pxc-client-6644d8898f-mzxrn +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok ++ set +o xtrace + source_primary=cross-site-source-pxc-0 ++ get_service_ip cross-site-source-pxc-0 ++ local service=cross-site-source-pxc-0 ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9O0dzXBCGB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.q1XJsayrSq +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.9O0dzXBCGB +++ cat /tmp/tmp.q1XJsayrSq +++ rm /tmp/tmp.9O0dzXBCGB /tmp/tmp.q1XJsayrSq +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MvlRkevj6A +++ mktemp ++ local LAST_ERR=/tmp/tmp.3C1E2y7nHJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MvlRkevj6A ++ cat /tmp/tmp.3C1E2y7nHJ ++ rm /tmp/tmp.MvlRkevj6A /tmp/tmp.3C1E2y7nHJ ++ return 0 ++ return + source_primary_endpoint=10.61.125.225 + desc 'patch source cluster with replicationChannels settings' + set +o xtrace ----------------------------------------------------------------------------------- patch source cluster with replicationChannels settings ----------------------------------------------------------------------------------- + kubectl_bin patch pxc cross-site-source --type=merge --patch '{"spec": {"pxc":{"replicationChannels": [{"name":"source_to_replica", "isSource": true}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.x2LzGK19t5 ++ mktemp + local LAST_ERR=/tmp/tmp.JrohphDtmr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-source --type=merge --patch '{"spec": {"pxc":{"replicationChannels": [{"name":"source_to_replica", "isSource": true}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.x2LzGK19t5 perconaxtradbcluster.pxc.percona.com/cross-site-source patched + cat /tmp/tmp.JrohphDtmr + rm /tmp/tmp.x2LzGK19t5 /tmp/tmp.JrohphDtmr + return 0 + desc 'patch main cluster secrets with replication user' + set +o xtrace ----------------------------------------------------------------------------------- patch main cluster secrets with replication user ----------------------------------------------------------------------------------- ++ echo -n new_password ++ base64 + kubectl_bin patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.ZMscrw6KSd ++ mktemp + local LAST_ERR=/tmp/tmp.TMJlf2Yg0g + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZMscrw6KSd secret/my-cluster-secrets patched + cat /tmp/tmp.TMJlf2Yg0g + rm /tmp/tmp.ZMscrw6KSd /tmp/tmp.TMJlf2Yg0g + return 0 + sleep 15 + wait_cluster_consistency cross-site-source 3 2 + local cluster_name=cross-site-source + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.91XtXW954r +++ mktemp ++ local LAST_ERR=/tmp/tmp.vRsZyW5wBD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.91XtXW954r ++ cat /tmp/tmp.vRsZyW5wBD ++ rm /tmp/tmp.91XtXW954r /tmp/tmp.vRsZyW5wBD ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vy7UTHOO2K +++ mktemp ++ local LAST_ERR=/tmp/tmp.eCR9LvEEDg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vy7UTHOO2K ++ cat /tmp/tmp.eCR9LvEEDg ++ rm /tmp/tmp.vy7UTHOO2K /tmp/tmp.eCR9LvEEDg ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.4kxcZMLCsS ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.A0PNzIAboU +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.4kxcZMLCsS +++++ cat /tmp/tmp.A0PNzIAboU +++++ rm /tmp/tmp.4kxcZMLCsS /tmp/tmp.A0PNzIAboU +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-haproxy ++++ return +++ local cluster_proxy=cross-site-source-haproxy +++ echo haproxy ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.W08aP4Pl9j +++ mktemp ++ local LAST_ERR=/tmp/tmp.6zFFWcYc0X ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.W08aP4Pl9j ++ cat /tmp/tmp.6zFFWcYc0X ++ rm /tmp/tmp.W08aP4Pl9j /tmp/tmp.6zFFWcYc0X ++ return 0 + [[ 2 == \2 ]] + desc 'write data to source cluster' + set +o xtrace ----------------------------------------------------------------------------------- write data to source cluster ----------------------------------------------------------------------------------- + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testSourceReplica (id int PRIMARY KEY);' '-h 10.61.125.225 -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testSourceReplica (id int PRIMARY KEY);' + local 'uri=-h 10.61.125.225 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EsA236ZUEn +++ mktemp ++ local LAST_ERR=/tmp/tmp.B5eIuvwpB3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EsA236ZUEn ++ cat /tmp/tmp.B5eIuvwpB3 ++ rm /tmp/tmp.EsA236ZUEn /tmp/tmp.B5eIuvwpB3 ++ return 0 + client_pod=pxc-client-6644d8898f-mzxrn + wait_pod pxc-client-6644d8898f-mzxrn + local pod=pxc-client-6644d8898f-mzxrn + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-mzxrn ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok + set +o xtrace + run_mysql 'INSERT myApp.testSourceReplica (id) VALUES (100700)' '-h 10.61.125.225 -uroot -proot_password' + local 'command=INSERT myApp.testSourceReplica (id) VALUES (100700)' + local 'uri=-h 10.61.125.225 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TUeX4exWll +++ mktemp ++ local LAST_ERR=/tmp/tmp.9d4TtGadAe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TUeX4exWll ++ cat /tmp/tmp.9d4TtGadAe ++ rm /tmp/tmp.TUeX4exWll /tmp/tmp.9d4TtGadAe ++ return 0 + client_pod=pxc-client-6644d8898f-mzxrn + wait_pod pxc-client-6644d8898f-mzxrn + local pod=pxc-client-6644d8898f-mzxrn + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-mzxrn ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok + set +o xtrace + yq 'del(.metadata)' - + kubectl_bin get secrets cross-site-source-ssl-internal -o yaml + yq '.metadata={"name": "cross-site-replica-ssl-internal"}' - ++ mktemp + local LAST_OUT=/tmp/tmp.z859nDZCJf ++ mktemp + local LAST_ERR=/tmp/tmp.h7QSQr8mOD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get secrets cross-site-source-ssl-internal -o yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.z859nDZCJf + cat /tmp/tmp.h7QSQr8mOD + rm /tmp/tmp.z859nDZCJf /tmp/tmp.h7QSQr8mOD + return 0 + desc 'take backup of source cluster' + set +o xtrace ----------------------------------------------------------------------------------- take backup of source cluster ----------------------------------------------------------------------------------- + run_backup cross-site-source backup-minio-source + local cluster=cross-site-source + local backup1=backup-minio-source + desc 'make backup backup-minio-source' + set +o xtrace ----------------------------------------------------------------------------------- make backup backup-minio-source ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/backup-minio-source.yml ++ mktemp + local LAST_OUT=/tmp/tmp.4ivNwNC3Ip ++ mktemp + local LAST_ERR=/tmp/tmp.4TGBmLBoB8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/backup-minio-source.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4ivNwNC3Ip perconaxtradbclusterbackup.pxc.percona.com/backup-minio-source created + cat /tmp/tmp.4TGBmLBoB8 + rm /tmp/tmp.4ivNwNC3Ip /tmp/tmp.4TGBmLBoB8 + return 0 + wait_backup backup-minio-source + local backup=backup-minio-source + local status=Succeeded + set +o xtrace backup-minio-source..............Succeeded + desc 'create replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- create replica cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-replica-20022 0 + local namespace=cross-site-replica-20022 + local skip_clean_namespace=0 + [[ 1 == 1 ]] + [[ -z 0 ]] + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-replica-20022' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-replica-20022 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-replica-20022 ++ mktemp + local LAST_OUT=/tmp/tmp.1JIyJy1ZHh ++ mktemp + local LAST_ERR=/tmp/tmp.qJtfCOAgYd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-replica-20022 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-replica-20022 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-replica-20022 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.1JIyJy1ZHh + cat /tmp/tmp.qJtfCOAgYd Error from server (NotFound): namespaces "cross-site-replica-20022" not found + rm /tmp/tmp.1JIyJy1ZHh /tmp/tmp.qJtfCOAgYd + return 1 + : + wait_for_delete namespace/cross-site-replica-20022 + local res=namespace/cross-site-replica-20022 + echo -n 'namespace/cross-site-replica-20022 - ' namespace/cross-site-replica-20022 - + set +o xtrace Error from server (NotFound): namespaces "cross-site-replica-20022" not found + desc 'create namespace cross-site-replica-20022' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-replica-20022 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-replica-20022 ++ mktemp + local LAST_OUT=/tmp/tmp.UalfJur0NA ++ mktemp + local LAST_ERR=/tmp/tmp.izlbFxke6D + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cross-site-replica-20022 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UalfJur0NA namespace/cross-site-replica-20022 created + cat /tmp/tmp.izlbFxke6D + rm /tmp/tmp.UalfJur0NA /tmp/tmp.izlbFxke6D + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.l7n9Mnj2AT +++ mktemp ++ local LAST_ERR=/tmp/tmp.4v8wAe4a5U ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.l7n9Mnj2AT ++ cat /tmp/tmp.4v8wAe4a5U ++ rm /tmp/tmp.l7n9Mnj2AT /tmp/tmp.4v8wAe4a5U ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3 --namespace=cross-site-replica-20022 ++ mktemp + local LAST_OUT=/tmp/tmp.qoH4KUijcs ++ mktemp + local LAST_ERR=/tmp/tmp.C6PE3SI6jm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3 --namespace=cross-site-replica-20022 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qoH4KUijcs Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3" modified. + cat /tmp/tmp.C6PE3SI6jm + rm /tmp/tmp.qoH4KUijcs /tmp/tmp.C6PE3SI6jm + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.yxaQGHyuqd ++ mktemp + local LAST_ERR=/tmp/tmp.ES0fRPFwus + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yxaQGHyuqd customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.ES0fRPFwus + rm /tmp/tmp.yxaQGHyuqd /tmp/tmp.ES0fRPFwus + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.RM6hpKU6JA ++ mktemp + local LAST_ERR=/tmp/tmp.eXzQdXRPqI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RM6hpKU6JA clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.eXzQdXRPqI + rm /tmp/tmp.RM6hpKU6JA /tmp/tmp.eXzQdXRPqI + return 0 + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1740-0a840b68^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/deploy/cw-operator.yaml + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - ++ mktemp + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + local LAST_OUT=/tmp/tmp.j2OSGBeYPq ++ mktemp + local LAST_ERR=/tmp/tmp.hgaURYOaHF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.j2OSGBeYPq deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.hgaURYOaHF + rm /tmp/tmp.j2OSGBeYPq /tmp/tmp.hgaURYOaHF + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.CaLlr8zmKz ++ mktemp + local LAST_ERR=/tmp/tmp.d2hutdlkOo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.CaLlr8zmKz + cat /tmp/tmp.d2hutdlkOo error: timed out waiting for the condition on pods/percona-xtradb-cluster-operator-6849457d9-zvl7h + rm /tmp/tmp.CaLlr8zmKz /tmp/tmp.d2hutdlkOo + return 1 + true ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.0acbSbpCZk +++ mktemp ++ local LAST_ERR=/tmp/tmp.wkqirwOSH7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0acbSbpCZk ++ cat /tmp/tmp.wkqirwOSH7 ++ rm /tmp/tmp.0acbSbpCZk /tmp/tmp.wkqirwOSH7 ++ return 0 + wait_pod percona-xtradb-cluster-operator-6849457d9-c7dd7 480 pxc-operator + local pod=percona-xtradb-cluster-operator-6849457d9-c7dd7 + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-6849457d9-c7dd7 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-6849457d9-c7dd7 condition met percona-xtradb-cluster-operator-6849457d9-c7dd7.Ok + sleep 3 + kubectl_bin apply -f /tmp/tmp.Su6U2NCsGL/replica-ssl-internal.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.Zjvc7OE5Jo ++ mktemp + local LAST_ERR=/tmp/tmp.A8HXQ8HBYS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /tmp/tmp.Su6U2NCsGL/replica-ssl-internal.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Zjvc7OE5Jo secret/cross-site-replica-ssl-internal created + cat /tmp/tmp.A8HXQ8HBYS + rm /tmp/tmp.Zjvc7OE5Jo /tmp/tmp.A8HXQ8HBYS + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.sQzdPqXC0f ++ mktemp + local LAST_ERR=/tmp/tmp.GWehKhPqZY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sQzdPqXC0f secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.GWehKhPqZY + rm /tmp/tmp.sQzdPqXC0f /tmp/tmp.GWehKhPqZY + return 0 + spinup_pxc cross-site-replica /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/cross-site-replica.yml + local cluster=cross-site-replica + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/cross-site-replica.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.yqLK3Hyaf2 ++ mktemp + local LAST_ERR=/tmp/tmp.IGTBKvoKMQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yqLK3Hyaf2 secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.IGTBKvoKMQ + rm /tmp/tmp.yqLK3Hyaf2 /tmp/tmp.IGTBKvoKMQ + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/client.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1740-0a840b68#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-27746~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.p3HFRgpFyf ++ mktemp + local LAST_ERR=/tmp/tmp.4iljGS9ChN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.p3HFRgpFyf deployment.apps/pxc-client created + cat /tmp/tmp.4iljGS9ChN + rm /tmp/tmp.p3HFRgpFyf /tmp/tmp.4iljGS9ChN + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/cross-site-replica.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/cross-site-replica.yml + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/cross-site-replica.yml + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1740-0a840b68#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-27746~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_OUT=/tmp/tmp.Pv7gnPV24N ++ mktemp + local LAST_ERR=/tmp/tmp.Sr47t3A6dc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Pv7gnPV24N perconaxtradbcluster.pxc.percona.com/cross-site-replica created + cat /tmp/tmp.Sr47t3A6dc + rm /tmp/tmp.Pv7gnPV24N /tmp/tmp.Sr47t3A6dc + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy cross-site-replica ++ local target_cluster=cross-site-replica +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.wIAg2NP0mO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UVLRL2wFvl +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.wIAg2NP0mO +++ cat /tmp/tmp.UVLRL2wFvl +++ rm /tmp/tmp.wIAg2NP0mO /tmp/tmp.UVLRL2wFvl +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo cross-site-replica-haproxy ++ return + local proxy=cross-site-replica-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-27746 ++ mktemp + local LAST_OUT=/tmp/tmp.e0TXPp99XZ ++ mktemp + local LAST_ERR=/tmp/tmp.97297YCCBT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-27746 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-27746 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-27746 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.e0TXPp99XZ + cat /tmp/tmp.97297YCCBT error: no matching resources found + rm /tmp/tmp.e0TXPp99XZ /tmp/tmp.97297YCCBT + return 1 + true + wait_for_running cross-site-replica-haproxy 1 + local name=cross-site-replica-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-haproxy-0 480 + local pod=cross-site-replica-haproxy-0 + local max_retry=480 + local ns= ++ echo cross-site-replica-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace Error from server (NotFound): pods "cross-site-replica-haproxy-0" not found cross-site-replica-haproxy-0...................................Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running cross-site-replica-pxc 3 + local name=cross-site-replica-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-0 480 + local pod=cross-site-replica-pxc-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo cross-site-replica-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/cross-site-replica-pxc-0 condition met cross-site-replica-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-1 480 + local pod=cross-site-replica-pxc-1 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo cross-site-replica-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/cross-site-replica-pxc-1 condition met cross-site-replica-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-2 480 + local pod=cross-site-replica-pxc-2 + local max_retry=480 + local ns= ++ echo cross-site-replica-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-replica-pxc-2 condition met cross-site-replica-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h cross-site-replica-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h cross-site-replica-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BW7GVpHTOJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.2AE6siH3a3 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BW7GVpHTOJ ++ cat /tmp/tmp.2AE6siH3a3 ++ rm /tmp/tmp.BW7GVpHTOJ /tmp/tmp.2AE6siH3a3 ++ return 0 + client_pod=pxc-client-6644d8898f-s8vpt + wait_pod pxc-client-6644d8898f-s8vpt + local pod=pxc-client-6644d8898f-s8vpt + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-s8vpt ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-s8vpt condition met pxc-client-6644d8898f-s8vpt.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h cross-site-replica-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h cross-site-replica-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4cPT0DhCBI +++ mktemp ++ local LAST_ERR=/tmp/tmp.rAmhsVxtdV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4cPT0DhCBI ++ cat /tmp/tmp.rAmhsVxtdV ++ rm /tmp/tmp.4cPT0DhCBI /tmp/tmp.rAmhsVxtdV ++ return 0 + client_pod=pxc-client-6644d8898f-s8vpt + wait_pod pxc-client-6644d8898f-s8vpt + local pod=pxc-client-6644d8898f-s8vpt + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6644d8898f-s8vpt ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-s8vpt condition met pxc-client-6644d8898f-s8vpt.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.C2fcx3sK1M +++ mktemp ++ local LAST_ERR=/tmp/tmp.1qH1bgLn7y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.C2fcx3sK1M ++ cat /tmp/tmp.1qH1bgLn7y ++ rm /tmp/tmp.C2fcx3sK1M /tmp/tmp.1qH1bgLn7y ++ return 0 + client_pod=pxc-client-6644d8898f-s8vpt + wait_pod pxc-client-6644d8898f-s8vpt + local pod=pxc-client-6644d8898f-s8vpt + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-s8vpt + local container= + set +o xtrace pod/pxc-client-6644d8898f-s8vpt condition met pxc-client-6644d8898f-s8vpt.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Su6U2NCsGL/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.Su6U2NCsGL/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ux20oIprex +++ mktemp ++ local LAST_ERR=/tmp/tmp.LUvYVVLb6V ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ux20oIprex ++ cat /tmp/tmp.LUvYVVLb6V ++ rm /tmp/tmp.ux20oIprex /tmp/tmp.LUvYVVLb6V ++ return 0 + client_pod=pxc-client-6644d8898f-s8vpt + wait_pod pxc-client-6644d8898f-s8vpt + local pod=pxc-client-6644d8898f-s8vpt + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-s8vpt ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-s8vpt condition met pxc-client-6644d8898f-s8vpt.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Su6U2NCsGL/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.Su6U2NCsGL/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.URQW9efKss +++ mktemp ++ local LAST_ERR=/tmp/tmp.NL8yiUVPW9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.URQW9efKss ++ cat /tmp/tmp.NL8yiUVPW9 ++ rm /tmp/tmp.URQW9efKss /tmp/tmp.NL8yiUVPW9 ++ return 0 + client_pod=pxc-client-6644d8898f-s8vpt + wait_pod pxc-client-6644d8898f-s8vpt + local pod=pxc-client-6644d8898f-s8vpt + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-s8vpt ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-s8vpt condition met pxc-client-6644d8898f-s8vpt.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Su6U2NCsGL/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.Su6U2NCsGL/select-1.sql ++ is_keyring_plugin_in_use cross-site-replica ++ local cluster=cross-site-replica ++ kubectl_bin exec -it cross-site-replica-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TLUZoTcmUr +++ mktemp ++ local LAST_ERR=/tmp/tmp.Uz5TxctYib ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it cross-site-replica-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TLUZoTcmUr ++ cat /tmp/tmp.Uz5TxctYib Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.TLUZoTcmUr /tmp/tmp.Uz5TxctYib ++ return 0 + '[' '' ']' + sleep 60 + desc 'restore backup from source cluster' + set +o xtrace ----------------------------------------------------------------------------------- restore backup from source cluster ----------------------------------------------------------------------------------- ++ kubectl_bin get -n cross-site-27746 pxc-backup backup-minio-source -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.clg89bp5T2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gzwW1w2QK4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get -n cross-site-27746 pxc-backup backup-minio-source -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.clg89bp5T2 ++ cat /tmp/tmp.gzwW1w2QK4 ++ rm /tmp/tmp.clg89bp5T2 /tmp/tmp.gzwW1w2QK4 ++ return 0 + destination=s3://operator-testing/cross-site-source-2024-06-28-07:56:50-full + kubectl_bin apply -f - + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-27746~ + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/restore-backup-minio.yml + /usr/bin/sed -e s~#cluster~cross-site-replica~ + /usr/bin/sed -e s~#destination~s3://operator-testing/cross-site-source-2024-06-28-07:56:50-full~ ++ mktemp + local LAST_OUT=/tmp/tmp.OgtS6HUeDP ++ mktemp + local LAST_ERR=/tmp/tmp.SOVCV3gPsO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OgtS6HUeDP perconaxtradbclusterrestore.pxc.percona.com/backup-minio created + cat /tmp/tmp.SOVCV3gPsO + rm /tmp/tmp.OgtS6HUeDP /tmp/tmp.SOVCV3gPsO + return 0 + wait_cluster_consistency cross-site-replica 3 2 + local cluster_name=cross-site-replica + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K2AAd2hemI +++ mktemp ++ local LAST_ERR=/tmp/tmp.cG8CY81K4J ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.K2AAd2hemI ++ cat /tmp/tmp.cG8CY81K4J ++ rm /tmp/tmp.K2AAd2hemI /tmp/tmp.cG8CY81K4J ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CVIMm2Hn7s +++ mktemp ++ local LAST_ERR=/tmp/tmp.ogNNqaPrsH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CVIMm2Hn7s ++ cat /tmp/tmp.ogNNqaPrsH ++ rm /tmp/tmp.CVIMm2Hn7s /tmp/tmp.ogNNqaPrsH ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.63k4BYuBmd +++ mktemp ++ local LAST_ERR=/tmp/tmp.q7gLtxsCOo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.63k4BYuBmd ++ cat /tmp/tmp.q7gLtxsCOo ++ rm /tmp/tmp.63k4BYuBmd /tmp/tmp.q7gLtxsCOo ++ return 0 + [[ paused == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gozVqmbQ4A +++ mktemp ++ local LAST_ERR=/tmp/tmp.UtMUXv8El9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gozVqmbQ4A ++ cat /tmp/tmp.UtMUXv8El9 ++ rm /tmp/tmp.gozVqmbQ4A /tmp/tmp.UtMUXv8El9 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.m6DI1vUY5t +++ mktemp ++ local LAST_ERR=/tmp/tmp.cDoatX4Wb4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.m6DI1vUY5t ++ cat /tmp/tmp.cDoatX4Wb4 ++ rm /tmp/tmp.m6DI1vUY5t /tmp/tmp.cDoatX4Wb4 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bXKReVay7L +++ mktemp ++ local LAST_ERR=/tmp/tmp.078IwaXAvU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bXKReVay7L ++ cat /tmp/tmp.078IwaXAvU ++ rm /tmp/tmp.bXKReVay7L /tmp/tmp.078IwaXAvU ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eymme9uOOo +++ mktemp ++ local LAST_ERR=/tmp/tmp.FUL2Y9YRVm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eymme9uOOo ++ cat /tmp/tmp.FUL2Y9YRVm ++ rm /tmp/tmp.eymme9uOOo /tmp/tmp.FUL2Y9YRVm ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RzhNRy5RWO +++ mktemp ++ local LAST_ERR=/tmp/tmp.iKwbZlIKRW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RzhNRy5RWO ++ cat /tmp/tmp.iKwbZlIKRW ++ rm /tmp/tmp.RzhNRy5RWO /tmp/tmp.iKwbZlIKRW ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yLBoqEEnD0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.XN9OFNBZNo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yLBoqEEnD0 ++ cat /tmp/tmp.XN9OFNBZNo ++ rm /tmp/tmp.yLBoqEEnD0 /tmp/tmp.XN9OFNBZNo ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 8 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KTT56IFptT +++ mktemp ++ local LAST_ERR=/tmp/tmp.uowG6eZu6i ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KTT56IFptT ++ cat /tmp/tmp.uowG6eZu6i ++ rm /tmp/tmp.KTT56IFptT /tmp/tmp.uowG6eZu6i ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 9 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JVaPLoAMyS +++ mktemp ++ local LAST_ERR=/tmp/tmp.SBLrPjlvUo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JVaPLoAMyS ++ cat /tmp/tmp.SBLrPjlvUo ++ rm /tmp/tmp.JVaPLoAMyS /tmp/tmp.SBLrPjlvUo ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 10 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u2jUn4OmIr +++ mktemp ++ local LAST_ERR=/tmp/tmp.qknX47ZKCZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u2jUn4OmIr ++ cat /tmp/tmp.qknX47ZKCZ ++ rm /tmp/tmp.u2jUn4OmIr /tmp/tmp.qknX47ZKCZ ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 11 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kWW8hyza1W +++ mktemp ++ local LAST_ERR=/tmp/tmp.RaruYH9I3M ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kWW8hyza1W ++ cat /tmp/tmp.RaruYH9I3M ++ rm /tmp/tmp.kWW8hyza1W /tmp/tmp.RaruYH9I3M ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0L9Wx9AaYf +++ mktemp ++ local LAST_ERR=/tmp/tmp.lakpX5gnbh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0L9Wx9AaYf ++ cat /tmp/tmp.lakpX5gnbh ++ rm /tmp/tmp.0L9Wx9AaYf /tmp/tmp.lakpX5gnbh ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.7UUgmg0C5I ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.k1HwVmwHAM +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.7UUgmg0C5I +++++ cat /tmp/tmp.k1HwVmwHAM +++++ rm /tmp/tmp.7UUgmg0C5I /tmp/tmp.k1HwVmwHAM +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-haproxy ++++ return +++ local cluster_proxy=cross-site-replica-haproxy +++ echo haproxy ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xWrDS1FYCN +++ mktemp ++ local LAST_ERR=/tmp/tmp.LDTmqWFWyK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xWrDS1FYCN ++ cat /tmp/tmp.LDTmqWFWyK ++ rm /tmp/tmp.xWrDS1FYCN /tmp/tmp.LDTmqWFWyK ++ return 0 + [[ 2 == \2 ]] + desc 'get replica cluster services endpoints' + set +o xtrace ----------------------------------------------------------------------------------- get replica cluster services endpoints ----------------------------------------------------------------------------------- ++ get_service_ip cross-site-replica-pxc-0 ++ local service=cross-site-replica-pxc-0 ++ grep -q NotFound ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QAPvK83x5a ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MovztyWWWs +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QAPvK83x5a +++ cat /tmp/tmp.MovztyWWWs +++ rm /tmp/tmp.QAPvK83x5a /tmp/tmp.MovztyWWWs +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BMKsP298Bh +++ mktemp ++ local LAST_ERR=/tmp/tmp.kdMLeokosx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BMKsP298Bh ++ cat /tmp/tmp.kdMLeokosx ++ rm /tmp/tmp.BMKsP298Bh /tmp/tmp.kdMLeokosx ++ return 0 ++ return + replica_endpoint0=10.61.123.204 ++ get_service_ip cross-site-replica-pxc-1 ++ local service=cross-site-replica-pxc-1 ++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DofIadGXAi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0o8WLwrgXI +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.DofIadGXAi +++ cat /tmp/tmp.0o8WLwrgXI +++ rm /tmp/tmp.DofIadGXAi /tmp/tmp.0o8WLwrgXI +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1oWEvK8ROK +++ mktemp ++ local LAST_ERR=/tmp/tmp.CdrsTIpft7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1oWEvK8ROK ++ cat /tmp/tmp.CdrsTIpft7 ++ rm /tmp/tmp.1oWEvK8ROK /tmp/tmp.CdrsTIpft7 ++ return 0 ++ return + replica_endpoint1=10.61.124.110 ++ get_service_ip cross-site-replica-pxc-2 ++ local service=cross-site-replica-pxc-2 ++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ja9IsQsBn0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.m4i6ftWTp9 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ja9IsQsBn0 +++ cat /tmp/tmp.m4i6ftWTp9 +++ rm /tmp/tmp.ja9IsQsBn0 /tmp/tmp.m4i6ftWTp9 +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OBSNnYI4Yh +++ mktemp ++ local LAST_ERR=/tmp/tmp.wcyyl5hQOB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OBSNnYI4Yh ++ cat /tmp/tmp.wcyyl5hQOB ++ rm /tmp/tmp.OBSNnYI4Yh /tmp/tmp.wcyyl5hQOB ++ return 0 ++ return + replica_endpoint2=10.61.125.34 ++ run_mysql 'SELECT @@hostname hostname;' '-h cross-site-replica-haproxy -p33062 -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h cross-site-replica-haproxy -p33062 -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.64bfh0AsXJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GsluGE3N7o +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.64bfh0AsXJ +++ cat /tmp/tmp.GsluGE3N7o +++ rm /tmp/tmp.64bfh0AsXJ /tmp/tmp.GsluGE3N7o +++ return 0 ++ client_pod=pxc-client-6644d8898f-s8vpt ++ wait_pod pxc-client-6644d8898f-s8vpt ++ local pod=pxc-client-6644d8898f-s8vpt ++ local max_retry=480 ++ local ns= +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ echo pxc-client-6644d8898f-s8vpt +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-6644d8898f-s8vpt condition met pxc-client-6644d8898f-s8vpt.Ok ++ set +o xtrace + replica_primary=cross-site-replica-pxc-0 ++ get_service_ip cross-site-replica-pxc-0 ++ local service=cross-site-replica-pxc-0 ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zlBAm2RMwp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vKyzguoKSQ +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zlBAm2RMwp +++ cat /tmp/tmp.vKyzguoKSQ +++ rm /tmp/tmp.zlBAm2RMwp /tmp/tmp.vKyzguoKSQ +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5skepnAYpn +++ mktemp ++ local LAST_ERR=/tmp/tmp.eSJM1gjLHI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5skepnAYpn ++ cat /tmp/tmp.eSJM1gjLHI ++ rm /tmp/tmp.5skepnAYpn /tmp/tmp.eSJM1gjLHI ++ return 0 ++ return + replica_primary_endpoint=10.61.123.204 + run_mysql 'DELETE FROM myApp.myApp WHERE id=100500' '-h 10.61.123.204 -uroot -proot_password' + local 'command=DELETE FROM myApp.myApp WHERE id=100500' + local 'uri=-h 10.61.123.204 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u5LSEYNWFX +++ mktemp ++ local LAST_ERR=/tmp/tmp.cUaizUn7T9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.u5LSEYNWFX ++ cat /tmp/tmp.cUaizUn7T9 ++ rm /tmp/tmp.u5LSEYNWFX /tmp/tmp.cUaizUn7T9 ++ return 0 + client_pod=pxc-client-6644d8898f-s8vpt + wait_pod pxc-client-6644d8898f-s8vpt + local pod=pxc-client-6644d8898f-s8vpt + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-s8vpt ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-s8vpt condition met pxc-client-6644d8898f-s8vpt.Ok + set +o xtrace + desc 'patch replica cluster with replicationChannels settings' + set +o xtrace ----------------------------------------------------------------------------------- patch replica cluster with replicationChannels settings ----------------------------------------------------------------------------------- + kubectl_bin patch pxc cross-site-replica --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "10.61.125.225", "port": 3306, "weight": 100},{"host": "10.61.122.184", "port": 3306, "weight": 100},{"host": "10.61.115.122", "port": 3306, "weight": 100}]}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.cqE2IvN9oc ++ mktemp + local LAST_ERR=/tmp/tmp.Kcoq8c8EEw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-replica --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "10.61.125.225", "port": 3306, "weight": 100},{"host": "10.61.122.184", "port": 3306, "weight": 100},{"host": "10.61.115.122", "port": 3306, "weight": 100}]}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cqE2IvN9oc perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + cat /tmp/tmp.Kcoq8c8EEw + rm /tmp/tmp.cqE2IvN9oc /tmp/tmp.Kcoq8c8EEw + return 0 + sleep 40 + desc 'patch replica cluster secrets with replication user' + set +o xtrace ----------------------------------------------------------------------------------- patch replica cluster secrets with replication user ----------------------------------------------------------------------------------- ++ echo -n new_password ++ base64 + kubectl_bin patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.DkgTWg0RyH ++ mktemp + local LAST_ERR=/tmp/tmp.MY74kMBwwi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DkgTWg0RyH secret/my-cluster-secrets patched + cat /tmp/tmp.MY74kMBwwi + rm /tmp/tmp.DkgTWg0RyH /tmp/tmp.MY74kMBwwi + return 0 + sleep 15 + wait_cluster_consistency cross-site-replica 3 2 + local cluster_name=cross-site-replica + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mf8Pp6Y3Vv +++ mktemp ++ local LAST_ERR=/tmp/tmp.Xztdx3miQm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mf8Pp6Y3Vv ++ cat /tmp/tmp.Xztdx3miQm ++ rm /tmp/tmp.mf8Pp6Y3Vv /tmp/tmp.Xztdx3miQm ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bQlArcEtT9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.SMinPifXiX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bQlArcEtT9 ++ cat /tmp/tmp.SMinPifXiX ++ rm /tmp/tmp.bQlArcEtT9 /tmp/tmp.SMinPifXiX ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.cSpO05nOTW ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.6bYGSbpcyH +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.cSpO05nOTW +++++ cat /tmp/tmp.6bYGSbpcyH +++++ rm /tmp/tmp.cSpO05nOTW /tmp/tmp.6bYGSbpcyH +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-haproxy ++++ return +++ local cluster_proxy=cross-site-replica-haproxy +++ echo haproxy ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PVnUzmrozr +++ mktemp ++ local LAST_ERR=/tmp/tmp.viR4OZffMz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PVnUzmrozr ++ cat /tmp/tmp.viR4OZffMz ++ rm /tmp/tmp.PVnUzmrozr /tmp/tmp.viR4OZffMz ++ return 0 + [[ 2 == \2 ]] + desc 'Check replication works between source -> replica' + set +o xtrace ----------------------------------------------------------------------------------- Check replication works between source -> replica ----------------------------------------------------------------------------------- + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 10.61.123.204 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.61.123.204 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-2-80.sql ']' + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 10.61.123.204 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.61.123.204 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.84RRAXU3pk +++ mktemp ++ local LAST_ERR=/tmp/tmp.vPwaeBZVpo ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.84RRAXU3pk ++ cat /tmp/tmp.vPwaeBZVpo ++ rm /tmp/tmp.84RRAXU3pk /tmp/tmp.vPwaeBZVpo ++ return 0 + client_pod=pxc-client-6644d8898f-s8vpt + wait_pod pxc-client-6644d8898f-s8vpt + local pod=pxc-client-6644d8898f-s8vpt + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-s8vpt ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-s8vpt condition met pxc-client-6644d8898f-s8vpt.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Su6U2NCsGL/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-2.sql /tmp/tmp.Su6U2NCsGL/select-2.sql + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 10.61.124.110 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.61.124.110 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-2-80.sql ']' + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 10.61.124.110 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.61.124.110 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.o1V2TdFgPH +++ mktemp ++ local LAST_ERR=/tmp/tmp.5IM9ZxV61P ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.o1V2TdFgPH ++ cat /tmp/tmp.5IM9ZxV61P ++ rm /tmp/tmp.o1V2TdFgPH /tmp/tmp.5IM9ZxV61P ++ return 0 + client_pod=pxc-client-6644d8898f-s8vpt + wait_pod pxc-client-6644d8898f-s8vpt + local pod=pxc-client-6644d8898f-s8vpt + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6644d8898f-s8vpt ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-s8vpt condition met pxc-client-6644d8898f-s8vpt.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Su6U2NCsGL/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-2.sql /tmp/tmp.Su6U2NCsGL/select-2.sql + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 10.61.125.34 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.61.125.34 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-2-80.sql ']' + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 10.61.125.34 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.61.125.34 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UrpfAiOZY2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4Vm9SRf6PC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UrpfAiOZY2 ++ cat /tmp/tmp.4Vm9SRf6PC ++ rm /tmp/tmp.UrpfAiOZY2 /tmp/tmp.4Vm9SRf6PC ++ return 0 + client_pod=pxc-client-6644d8898f-s8vpt + wait_pod pxc-client-6644d8898f-s8vpt + local pod=pxc-client-6644d8898f-s8vpt + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-s8vpt ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-s8vpt condition met pxc-client-6644d8898f-s8vpt.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Su6U2NCsGL/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-2.sql /tmp/tmp.Su6U2NCsGL/select-2.sql + run_backup cross-site-replica backup-minio-replica + local cluster=cross-site-replica + local backup1=backup-minio-replica + desc 'make backup backup-minio-replica' + set +o xtrace ----------------------------------------------------------------------------------- make backup backup-minio-replica ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/backup-minio-replica.yml ++ mktemp + local LAST_OUT=/tmp/tmp.smITkQf5dU ++ mktemp + local LAST_ERR=/tmp/tmp.DNNiZDYrXj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/backup-minio-replica.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.smITkQf5dU perconaxtradbclusterbackup.pxc.percona.com/backup-minio-replica created + cat /tmp/tmp.DNNiZDYrXj + rm /tmp/tmp.smITkQf5dU /tmp/tmp.DNNiZDYrXj + return 0 + wait_backup backup-minio-replica + local backup=backup-minio-replica + local status=Succeeded + set +o xtrace backup-minio-replica.............Succeeded + desc 'Switch clusters over' + set +o xtrace ----------------------------------------------------------------------------------- Switch clusters over ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.XCyD7jV17e +++ mktemp ++ local LAST_ERR=/tmp/tmp.mFfdgB0Kkg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XCyD7jV17e ++ cat /tmp/tmp.mFfdgB0Kkg ++ rm /tmp/tmp.XCyD7jV17e /tmp/tmp.mFfdgB0Kkg ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3 --namespace=cross-site-27746 ++ mktemp + local LAST_OUT=/tmp/tmp.pw8hW2B1Zg ++ mktemp + local LAST_ERR=/tmp/tmp.b9yLszNW7M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3 --namespace=cross-site-27746 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pw8hW2B1Zg Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3" modified. + cat /tmp/tmp.b9yLszNW7M + rm /tmp/tmp.pw8hW2B1Zg /tmp/tmp.b9yLszNW7M + return 0 + desc 'rebuild source cluster' + set +o xtrace ----------------------------------------------------------------------------------- rebuild source cluster ----------------------------------------------------------------------------------- ++ kubectl_bin get -n cross-site-replica-20022 pxc-backup backup-minio-replica -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6YRFWL3Svc +++ mktemp ++ local LAST_ERR=/tmp/tmp.iXER6F2RZc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get -n cross-site-replica-20022 pxc-backup backup-minio-replica -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6YRFWL3Svc ++ cat /tmp/tmp.iXER6F2RZc ++ rm /tmp/tmp.6YRFWL3Svc /tmp/tmp.iXER6F2RZc ++ return 0 + destination=s3://operator-testing/cross-site-replica-2024-06-28-08:12:25-full + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/conf/restore-backup-minio.yml + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-27746~ + /usr/bin/sed -e s~#destination~s3://operator-testing/cross-site-replica-2024-06-28-08:12:25-full~ + kubectl_bin apply -f - + /usr/bin/sed -e s~#cluster~cross-site-source~ ++ mktemp + local LAST_OUT=/tmp/tmp.BFriNhPuCW ++ mktemp + local LAST_ERR=/tmp/tmp.KSNkAAI7sm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BFriNhPuCW perconaxtradbclusterrestore.pxc.percona.com/backup-minio created + cat /tmp/tmp.KSNkAAI7sm + rm /tmp/tmp.BFriNhPuCW /tmp/tmp.KSNkAAI7sm + return 0 + wait_cluster_consistency cross-site-source 3 2 + local cluster_name=cross-site-source + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zm1lOJhsf6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.UdcNXKq0MV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zm1lOJhsf6 ++ cat /tmp/tmp.UdcNXKq0MV ++ rm /tmp/tmp.zm1lOJhsf6 /tmp/tmp.UdcNXKq0MV ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 0 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w6Wv8O939B +++ mktemp ++ local LAST_ERR=/tmp/tmp.p83UIcJsdp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w6Wv8O939B ++ cat /tmp/tmp.p83UIcJsdp ++ rm /tmp/tmp.w6Wv8O939B /tmp/tmp.p83UIcJsdp ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 1 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1uHco3MkZL +++ mktemp ++ local LAST_ERR=/tmp/tmp.3PXPd3RMBk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1uHco3MkZL ++ cat /tmp/tmp.3PXPd3RMBk ++ rm /tmp/tmp.1uHco3MkZL /tmp/tmp.3PXPd3RMBk ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 2 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.C9gYIHc0zW +++ mktemp ++ local LAST_ERR=/tmp/tmp.MlxNq6qjaW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.C9gYIHc0zW ++ cat /tmp/tmp.MlxNq6qjaW ++ rm /tmp/tmp.C9gYIHc0zW /tmp/tmp.MlxNq6qjaW ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 3 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n8tEXe8LOg +++ mktemp ++ local LAST_ERR=/tmp/tmp.Zcot3ZrSSp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.n8tEXe8LOg ++ cat /tmp/tmp.Zcot3ZrSSp ++ rm /tmp/tmp.n8tEXe8LOg /tmp/tmp.Zcot3ZrSSp ++ return 0 + [[ paused == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 4 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pkDGQBAbA7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4mwlYgeg2a ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pkDGQBAbA7 ++ cat /tmp/tmp.4mwlYgeg2a ++ rm /tmp/tmp.pkDGQBAbA7 /tmp/tmp.4mwlYgeg2a ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 5 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3b5XVw28kX +++ mktemp ++ local LAST_ERR=/tmp/tmp.bpCvDkHza0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3b5XVw28kX ++ cat /tmp/tmp.bpCvDkHza0 ++ rm /tmp/tmp.3b5XVw28kX /tmp/tmp.bpCvDkHza0 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 6 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yfuTJiv39P +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZPeIZ6EAbB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yfuTJiv39P ++ cat /tmp/tmp.ZPeIZ6EAbB ++ rm /tmp/tmp.yfuTJiv39P /tmp/tmp.ZPeIZ6EAbB ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 7 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FPvw3JF5z7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.PstjZFXcwf ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FPvw3JF5z7 ++ cat /tmp/tmp.PstjZFXcwf ++ rm /tmp/tmp.FPvw3JF5z7 /tmp/tmp.PstjZFXcwf ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 8 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fhr67Eh5Zh +++ mktemp ++ local LAST_ERR=/tmp/tmp.8Athzrd5Qt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fhr67Eh5Zh ++ cat /tmp/tmp.8Athzrd5Qt ++ rm /tmp/tmp.fhr67Eh5Zh /tmp/tmp.8Athzrd5Qt ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 9 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tkd3MVmCL4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.V40IcaYMxv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tkd3MVmCL4 ++ cat /tmp/tmp.V40IcaYMxv ++ rm /tmp/tmp.tkd3MVmCL4 /tmp/tmp.V40IcaYMxv ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 10 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XeRzOvKQKC +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZfJfgOi9DA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XeRzOvKQKC ++ cat /tmp/tmp.ZfJfgOi9DA ++ rm /tmp/tmp.XeRzOvKQKC /tmp/tmp.ZfJfgOi9DA ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 11 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8hkrmNaQyW +++ mktemp ++ local LAST_ERR=/tmp/tmp.MGRPFscFNp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8hkrmNaQyW ++ cat /tmp/tmp.MGRPFscFNp ++ rm /tmp/tmp.8hkrmNaQyW /tmp/tmp.MGRPFscFNp ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 12 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZThfBYnJta +++ mktemp ++ local LAST_ERR=/tmp/tmp.kc2dy4AHOr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZThfBYnJta ++ cat /tmp/tmp.kc2dy4AHOr ++ rm /tmp/tmp.ZThfBYnJta /tmp/tmp.kc2dy4AHOr ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 13 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VBtkMsJVjD +++ mktemp ++ local LAST_ERR=/tmp/tmp.s4LNcDo4dH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VBtkMsJVjD ++ cat /tmp/tmp.s4LNcDo4dH ++ rm /tmp/tmp.VBtkMsJVjD /tmp/tmp.s4LNcDo4dH ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 + [[ 14 -ge 36 ]] + let i+=1 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.e8zxCRXp9O +++ mktemp ++ local LAST_ERR=/tmp/tmp.FxvGAJSqVw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.e8zxCRXp9O ++ cat /tmp/tmp.FxvGAJSqVw ++ rm /tmp/tmp.e8zxCRXp9O /tmp/tmp.FxvGAJSqVw ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.F00VLoalQR +++ mktemp ++ local LAST_ERR=/tmp/tmp.nGh8rOeO2v ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.F00VLoalQR ++ cat /tmp/tmp.nGh8rOeO2v ++ rm /tmp/tmp.F00VLoalQR /tmp/tmp.nGh8rOeO2v ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.ese1HSGb46 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Lv4IpLkLwt +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.ese1HSGb46 +++++ cat /tmp/tmp.Lv4IpLkLwt +++++ rm /tmp/tmp.ese1HSGb46 /tmp/tmp.Lv4IpLkLwt +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-haproxy ++++ return +++ local cluster_proxy=cross-site-source-haproxy +++ echo haproxy ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DrgUJtFZEA +++ mktemp ++ local LAST_ERR=/tmp/tmp.sxKdes3IBW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DrgUJtFZEA ++ cat /tmp/tmp.sxKdes3IBW ++ rm /tmp/tmp.DrgUJtFZEA /tmp/tmp.sxKdes3IBW ++ return 0 + [[ 2 == \2 ]] + run_mysql 'DELETE FROM myApp.myApp WHERE id=100500' '-h 10.61.125.225 -uroot -proot_password' + local 'command=DELETE FROM myApp.myApp WHERE id=100500' + local 'uri=-h 10.61.125.225 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TWxqMNjAwR +++ mktemp ++ local LAST_ERR=/tmp/tmp.dQ1SFcdndy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TWxqMNjAwR ++ cat /tmp/tmp.dQ1SFcdndy ++ rm /tmp/tmp.TWxqMNjAwR /tmp/tmp.dQ1SFcdndy ++ return 0 + client_pod=pxc-client-6644d8898f-mzxrn + wait_pod pxc-client-6644d8898f-mzxrn + local pod=pxc-client-6644d8898f-mzxrn + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-mzxrn + local container= + set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok + set +o xtrace + run_mysql 'DELETE FROM myApp.testSourceReplica WHERE id=100700' '-h 10.61.125.225 -uroot -proot_password' + local 'command=DELETE FROM myApp.testSourceReplica WHERE id=100700' + local 'uri=-h 10.61.125.225 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.clcfgejxaJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.oYDslsnxOS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.clcfgejxaJ ++ cat /tmp/tmp.oYDslsnxOS ++ rm /tmp/tmp.clcfgejxaJ /tmp/tmp.oYDslsnxOS ++ return 0 + client_pod=pxc-client-6644d8898f-mzxrn + wait_pod pxc-client-6644d8898f-mzxrn + local pod=pxc-client-6644d8898f-mzxrn + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-mzxrn ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok + set +o xtrace + desc 'configure old replica as source' + set +o xtrace ----------------------------------------------------------------------------------- configure old replica as source ----------------------------------------------------------------------------------- + kubectl patch pxc cross-site-replica -n cross-site-replica-20022 --type=json '-p=[{'\''op'\'': '\''remove'\'', '\''path'\'': '\''/spec/pxc/replicationChannels/0'\''}]' perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + kubectl_bin patch pxc cross-site-replica -n cross-site-replica-20022 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": true}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.ZbRBNxVjuj ++ mktemp + local LAST_ERR=/tmp/tmp.GOowix6BsC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-replica -n cross-site-replica-20022 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": true}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZbRBNxVjuj perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + cat /tmp/tmp.GOowix6BsC + rm /tmp/tmp.ZbRBNxVjuj /tmp/tmp.GOowix6BsC + return 0 + desc 'configure old source as replica' + set +o xtrace ----------------------------------------------------------------------------------- configure old source as replica ----------------------------------------------------------------------------------- + kubectl patch pxc cross-site-source -n cross-site-27746 --type=json '-p=[{'\''op'\'': '\''remove'\'', '\''path'\'': '\''/spec/pxc/replicationChannels/0'\''}]' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + kubectl_bin patch pxc cross-site-source -n cross-site-27746 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "10.61.123.204", "port": 3306, "weight": 100},{"host": "10.61.124.110", "port": 3306, "weight": 100},{"host": "10.61.125.34", "port": 3306, "weight": 100}]}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.vFKQ8BM3Bz ++ mktemp + local LAST_ERR=/tmp/tmp.FZLDPHHkvD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-source -n cross-site-27746 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "10.61.123.204", "port": 3306, "weight": 100},{"host": "10.61.124.110", "port": 3306, "weight": 100},{"host": "10.61.125.34", "port": 3306, "weight": 100}]}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vFKQ8BM3Bz perconaxtradbcluster.pxc.percona.com/cross-site-source patched + cat /tmp/tmp.FZLDPHHkvD + rm /tmp/tmp.vFKQ8BM3Bz /tmp/tmp.FZLDPHHkvD + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.eIUuSuGfhl +++ mktemp ++ local LAST_ERR=/tmp/tmp.KXzemjRt6R ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eIUuSuGfhl ++ cat /tmp/tmp.KXzemjRt6R ++ rm /tmp/tmp.eIUuSuGfhl /tmp/tmp.KXzemjRt6R ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3 --namespace=cross-site-replica-20022 ++ mktemp + local LAST_OUT=/tmp/tmp.HVCicIIKZR ++ mktemp + local LAST_ERR=/tmp/tmp.FeZi7B641C + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3 --namespace=cross-site-replica-20022 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HVCicIIKZR Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3" modified. + cat /tmp/tmp.FeZi7B641C + rm /tmp/tmp.HVCicIIKZR /tmp/tmp.FeZi7B641C + return 0 + desc 'Write data to replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- Write data to replica cluster ----------------------------------------------------------------------------------- + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testReplicaSource (id int PRIMARY KEY);' '-h 10.61.123.204 -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testReplicaSource (id int PRIMARY KEY);' + local 'uri=-h 10.61.123.204 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xQfvWRyfP4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ozzJ07tpx4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xQfvWRyfP4 ++ cat /tmp/tmp.ozzJ07tpx4 ++ rm /tmp/tmp.xQfvWRyfP4 /tmp/tmp.ozzJ07tpx4 ++ return 0 + client_pod=pxc-client-6644d8898f-kn7rq + wait_pod pxc-client-6644d8898f-kn7rq + local pod=pxc-client-6644d8898f-kn7rq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-kn7rq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-kn7rq condition met pxc-client-6644d8898f-kn7rq.Ok + set +o xtrace + run_mysql 'INSERT myApp.testReplicaSource (id) VALUES (100800)' '-h 10.61.123.204 -uroot -proot_password' + local 'command=INSERT myApp.testReplicaSource (id) VALUES (100800)' + local 'uri=-h 10.61.123.204 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w1b1FjwZJq +++ mktemp ++ local LAST_ERR=/tmp/tmp.SjM35dVykG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w1b1FjwZJq ++ cat /tmp/tmp.SjM35dVykG ++ rm /tmp/tmp.w1b1FjwZJq /tmp/tmp.SjM35dVykG ++ return 0 + client_pod=pxc-client-6644d8898f-kn7rq + wait_pod pxc-client-6644d8898f-kn7rq + local pod=pxc-client-6644d8898f-kn7rq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-kn7rq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-kn7rq condition met pxc-client-6644d8898f-kn7rq.Ok + set +o xtrace + sleep 15 + desc 'Check replication works between replica -> source' + set +o xtrace ----------------------------------------------------------------------------------- Check replication works between replica -> source ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.QEOthfkHAn +++ mktemp ++ local LAST_ERR=/tmp/tmp.scEtIYkcMT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QEOthfkHAn ++ cat /tmp/tmp.scEtIYkcMT ++ rm /tmp/tmp.QEOthfkHAn /tmp/tmp.scEtIYkcMT ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3 --namespace=cross-site-27746 ++ mktemp + local LAST_OUT=/tmp/tmp.tCV0dnGUP2 ++ mktemp + local LAST_ERR=/tmp/tmp.sit1JE0PZ5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3 --namespace=cross-site-27746 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tCV0dnGUP2 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1740-0a840b68-3-cluster3" modified. + cat /tmp/tmp.sit1JE0PZ5 + rm /tmp/tmp.tCV0dnGUP2 /tmp/tmp.sit1JE0PZ5 + return 0 + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 10.61.125.225 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.61.125.225 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-3-80.sql ']' + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 10.61.125.225 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.61.125.225 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pwMhhF9Yt9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.zG3tWllyO7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pwMhhF9Yt9 ++ cat /tmp/tmp.zG3tWllyO7 ++ rm /tmp/tmp.pwMhhF9Yt9 /tmp/tmp.zG3tWllyO7 ++ return 0 + client_pod=pxc-client-6644d8898f-mzxrn + wait_pod pxc-client-6644d8898f-mzxrn + local pod=pxc-client-6644d8898f-mzxrn + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-mzxrn ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Su6U2NCsGL/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-3.sql /tmp/tmp.Su6U2NCsGL/select-3.sql + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 10.61.122.184 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.61.122.184 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-3-80.sql ']' + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 10.61.122.184 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.61.122.184 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Gi4HZbKKTO +++ mktemp ++ local LAST_ERR=/tmp/tmp.LRIk4mYs41 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Gi4HZbKKTO ++ cat /tmp/tmp.LRIk4mYs41 ++ rm /tmp/tmp.Gi4HZbKKTO /tmp/tmp.LRIk4mYs41 ++ return 0 + client_pod=pxc-client-6644d8898f-mzxrn + wait_pod pxc-client-6644d8898f-mzxrn + local pod=pxc-client-6644d8898f-mzxrn + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-mzxrn ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Su6U2NCsGL/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-3.sql /tmp/tmp.Su6U2NCsGL/select-3.sql + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 10.61.115.122 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.61.115.122 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-3-80.sql ']' + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 10.61.115.122 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.61.115.122 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XI4itLH1fC +++ mktemp ++ local LAST_ERR=/tmp/tmp.lhiz6MREEY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XI4itLH1fC ++ cat /tmp/tmp.lhiz6MREEY ++ rm /tmp/tmp.XI4itLH1fC /tmp/tmp.lhiz6MREEY ++ return 0 + client_pod=pxc-client-6644d8898f-mzxrn + wait_pod pxc-client-6644d8898f-mzxrn + local pod=pxc-client-6644d8898f-mzxrn + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-mzxrn ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-mzxrn condition met pxc-client-6644d8898f-mzxrn.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.Su6U2NCsGL/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1740/e2e-tests/cross-site/compare/select-3.sql /tmp/tmp.Su6U2NCsGL/select-3.sql + destroy cross-site-27746 true + local namespace=cross-site-27746 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + tee /tmp/tmp.Su6U2NCsGL/operator.log + grep -v 'get backup status: Job.batch' + sort -u +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator + grep -v level=info + grep -v 'the object has been modified' ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.8lQMsBHZEl +++ mktemp ++ local LAST_ERR=/tmp/tmp.qFgtSo8AuZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8lQMsBHZEl ++ cat /tmp/tmp.qFgtSo8AuZ ++ rm /tmp/tmp.8lQMsBHZEl /tmp/tmp.qFgtSo8AuZ ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-6849457d9-c7dd7 ++ mktemp + local LAST_OUT=/tmp/tmp.SW1wFPC2c3 ++ mktemp + local LAST_ERR=/tmp/tmp.1u43ZTmQDI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-6849457d9-c7dd7 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SW1wFPC2c3 + cat /tmp/tmp.1u43ZTmQDI + rm /tmp/tmp.SW1wFPC2c3 /tmp/tmp.1u43ZTmQDI + return 0 2024-06-28T07:47:43.556Z INFO setup Manager starting up {"gitCommit": "0a840b68490b2f8881fb749474303f7fb8a1239d", "gitBranch": "PR-1740-0a840b68", "buildTime": "2024-06-28T06:58:02Z", "goVersion": "go1.22.4", "os": "linux", "arch": "amd64"} 2024-06-28T07:47:43.556Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.26.15-gke.1436000"} 2024-06-28T07:47:43.610Z INFO setup Registering Components. 2024-06-28T07:47:48.213Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2024-06-28T07:47:48.216Z INFO controller-runtime.metrics Starting metrics server 2024-06-28T07:47:48.216Z INFO setup Starting the Cmd. 2024-06-28T07:47:48.217Z INFO controller-runtime.certwatcher Starting certificate watcher 2024-06-28T07:47:48.217Z INFO controller-runtime.certwatcher Updated current TLS certificate 2024-06-28T07:47:48.217Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2024-06-28T07:47:48.217Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2024-06-28T07:47:48.217Z INFO controller-runtime.webhook Starting webhook server 2024-06-28T07:47:48.217Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2024-06-28T07:47:48.318Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2024-06-28T07:47:48.331Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2024-06-28T07:47:48.332Z DEBUG events percona-xtradb-cluster-operator-6849457d9-c7dd7_0c158c91-fe20-45c0-84ba-e150208f9bd1 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"5e2f0e36-640a-4099-aaba-dfc6afd138ef","apiVersion":"coordination.k8s.io/v1","resourceVersion":"24143"}, "reason": "LeaderElection"} 2024-06-28T07:47:48.332Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2024-06-28T07:47:48.332Z INFO Starting Controller {"controller": "pxc-controller"} 2024-06-28T07:47:48.332Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2024-06-28T07:47:48.332Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2024-06-28T07:47:48.332Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"} 2024-06-28T07:47:48.332Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2024-06-28T07:47:48.438Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2024-06-28T07:47:48.438Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2024-06-28T07:47:48.438Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2024-06-28T07:49:30.805Z INFO Set CR version {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "034d94d5-f097-451f-8f26-3f7571a461b5", "version": "1.15.0"} 2024-06-28T07:49:38.853Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 44cfafd5-0642-4002-8e83-62d612d126f9 2024-06-28T07:50:51.292Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9", "user": "operator"} 2024-06-28T07:50:51.334Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9", "user": "monitor"} 2024-06-28T07:50:51.440Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9"} 2024-06-28T07:50:51.482Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9"} 2024-06-28T07:50:51.519Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9", "user": "xtrabackup"} 2024-06-28T07:50:51.579Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9"} 2024-06-28T07:50:51.618Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9", "user": "replication"} 2024-06-28T07:50:53.723Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.61.123.52:3306: connect: connection refused"} 2024-06-28T07:53:20.264Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "b6326dbc-30dc-476f-bf63-c27bdab1b0e0", "user": "root"} 2024-06-28T07:53:20.618Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "b6326dbc-30dc-476f-bf63-c27bdab1b0e0", "new version": "8.0.36-28.1"} 2024-06-28T07:56:04.474Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "7acecdc5-6c20-48d5-8d24-6c66e8d9f849", "user": "replication"} 2024-06-28T07:56:04.525Z INFO Password updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "7acecdc5-6c20-48d5-8d24-6c66e8d9f849", "user": "replication"} 2024-06-28T07:56:04.538Z INFO MySQL init secret created {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "7acecdc5-6c20-48d5-8d24-6c66e8d9f849", "secret": "cross-site-source-mysql-init", "user": "replication"} 2024-06-28T07:56:04.551Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "7acecdc5-6c20-48d5-8d24-6c66e8d9f849", "user": "replication"} 2024-06-28T07:56:04.583Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "7acecdc5-6c20-48d5-8d24-6c66e8d9f849", "user": "replication"} 2024-06-28T07:56:50.120Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "cross-site-27746", "name": "backup-minio-source", "reconcileID": "d632aea5-281c-4e9f-af23-4866f86243e6", "Namespace": "cross-site-27746", "Name": "xb-backup-minio-source"} 2024-06-28T07:59:38.239Z INFO Set CR version {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "405fb30e-30f4-452b-b13d-11848f993bd7", "version": "1.15.0"} 2024-06-28T07:59:48.335Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 8bfe22cf-d3fa-48a5-b8c9-eec3fe48bab0 2024-06-28T08:01:06.085Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296", "user": "operator"} 2024-06-28T08:01:06.134Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296", "user": "monitor"} 2024-06-28T08:01:06.235Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296"} 2024-06-28T08:01:06.271Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296"} 2024-06-28T08:01:06.309Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296", "user": "xtrabackup"} 2024-06-28T08:01:06.366Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296"} 2024-06-28T08:01:06.403Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296", "user": "replication"} 2024-06-28T08:03:27.674Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "2117d8a7-1a3e-450d-b582-120b49b14d0f", "user": "root"} 2024-06-28T08:03:27.918Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "2117d8a7-1a3e-450d-b582-120b49b14d0f", "new version": "8.0.36-28.1"} 2024-06-28T08:05:56.781Z INFO backup restore request {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-20022", "name": "backup-minio", "reconcileID": "96262dcb-374a-43bd-bc4f-bce7b802699b"} 2024-06-28T08:05:56.834Z INFO stopping cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-20022", "name": "backup-minio", "reconcileID": "96262dcb-374a-43bd-bc4f-bce7b802699b", "cluster": "cross-site-replica"} 2024-06-28T08:06:42.990Z INFO starting restore {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-20022", "name": "backup-minio", "reconcileID": "96262dcb-374a-43bd-bc4f-bce7b802699b", "cluster": "cross-site-replica", "backup": ""} 2024-06-28T08:07:06.056Z INFO starting cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-20022", "name": "backup-minio", "reconcileID": "96262dcb-374a-43bd-bc4f-bce7b802699b", "cluster": "cross-site-replica"} 2024-06-28T08:10:11.235Z INFO You can view xtrabackup log: 2024-06-28T08:10:54.474Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "6083c2bc-125c-49eb-8d7f-9a98f0810298", "pod": "cross-site-replica-pxc-0"} 2024-06-28T08:10:54.495Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "6083c2bc-125c-49eb-8d7f-9a98f0810298", "pod": "cross-site-replica-pxc-1"} 2024-06-28T08:10:54.522Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "6083c2bc-125c-49eb-8d7f-9a98f0810298", "pod": "cross-site-replica-pxc-2"} 2024-06-28T08:10:54.571Z INFO Replication pod has changed {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "6083c2bc-125c-49eb-8d7f-9a98f0810298", "new replication pod": "cross-site-replica-pxc-0"} 2024-06-28T08:10:56.101Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "45c7917c-4e0a-44ed-a72a-15fe7d0b4185", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:00.497Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "d52d7e5b-ad8e-4874-a94e-9413dc4ee4c1", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:05.967Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "e269ccb1-51ce-4b5b-bb63-c6e83766691e", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:11.426Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "0f3fd240-3d33-4025-9827-d62c4e7c69c4", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:16.866Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "bae0579c-f9a8-4acc-ad01-872db849695d", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:23.084Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "2b03823d-3035-43ca-b615-91005cd63027", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:28.717Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "07891407-b2c8-4747-b9ed-f6ee2bb62a93", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:34.220Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "84c1c78e-9e8e-4e18-b939-09835b912107", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:39.540Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "57160a3d-2fea-4eda-9c8f-612af742fbcd", "user": "replication"} 2024-06-28T08:11:39.576Z INFO Password updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "57160a3d-2fea-4eda-9c8f-612af742fbcd", "user": "replication"} 2024-06-28T08:11:39.586Z INFO MySQL init secret created {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "57160a3d-2fea-4eda-9c8f-612af742fbcd", "secret": "cross-site-replica-mysql-init", "user": "replication"} 2024-06-28T08:11:39.601Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "57160a3d-2fea-4eda-9c8f-612af742fbcd", "user": "replication"} 2024-06-28T08:11:39.677Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "57160a3d-2fea-4eda-9c8f-612af742fbcd", "user": "replication"} 2024-06-28T08:11:39.858Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "57160a3d-2fea-4eda-9c8f-612af742fbcd", "channel": "source_to_replica", "Last_IO_Error": ""} 2024-06-28T08:12:25.420Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "cross-site-replica-20022", "name": "backup-minio-replica", "reconcileID": "c386b13a-ad73-4f58-ae87-35cf057a2589", "Namespace": "cross-site-replica-20022", "Name": "xb-backup-minio-replica"} 2024-06-28T08:12:59.582Z INFO backup restore request {"controller": "pxcrestore-controller", "namespace": "cross-site-27746", "name": "backup-minio", "reconcileID": "6fe2e644-59c4-4d52-821f-e51151241789"} 2024-06-28T08:12:59.715Z INFO stopping cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-27746", "name": "backup-minio", "reconcileID": "6fe2e644-59c4-4d52-821f-e51151241789", "cluster": "cross-site-source"} 2024-06-28T08:13:03.778Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "3469ee8b-7394-4379-9dde-df46d93fc466", "err": "failed to ensure cluster readonly status: connect to pod cross-site-replica-pxc-1: dial tcp 10.204.121.19:33062: connect: connection refused"} 2024-06-28T08:13:39.928Z INFO starting restore {"controller": "pxcrestore-controller", "namespace": "cross-site-27746", "name": "backup-minio", "reconcileID": "6fe2e644-59c4-4d52-821f-e51151241789", "cluster": "cross-site-source", "backup": ""} 2024-06-28T08:14:11.432Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "3ed77b41-1f4d-4065-8d0d-d17e6cf47af2", "err": "get primary pxc pod: failed to get proxy connection: dial tcp: lookup cross-site-replica-haproxy.cross-site-replica-20022: i/o timeout"} 2024-06-28T08:14:27.820Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "3087acb8-65cf-4f30-8e5e-696711d93d48", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:28.250Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "87cf6fa7-052a-4c26-afa2-008327c2d3c7", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:33.163Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "aec7abee-a2be-497a-9721-cf99e9da1d21", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:33.750Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "3d603911-a7d3-45cf-a6d0-f64c4a96acf7", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:38.509Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "1b1720f8-f61d-417f-ad4b-23ab2d0fa1e5", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:40.058Z INFO starting cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-27746", "name": "backup-minio", "reconcileID": "6fe2e644-59c4-4d52-821f-e51151241789", "cluster": "cross-site-source"} 2024-06-28T08:14:44.126Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "d593e9a0-e6a8-4e9b-885d-d69d5e50d349", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:49.517Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "6938b603-199c-4601-b8ed-1e68174437cd", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:54.772Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "557bbc80-f625-47b4-9ecc-6e9ed5e50b0b", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:00.111Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "7ebf22ce-bb9d-4837-8e15-417665d2287c", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:05.512Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "9d77d5d2-36c1-4253-9f58-38a7d7693108", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:10.826Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "17400f69-c9aa-4e73-9314-d67a6ea75003", "pod": "cross-site-replica-pxc-1"} 2024-06-28T08:15:10.842Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "17400f69-c9aa-4e73-9314-d67a6ea75003", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:11.867Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "684d7998-baae-41aa-b30f-c72a55ae9cad", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:16.307Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "88ba3434-b850-45d5-b5e8-ff510372810c", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:21.909Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "6ee3484f-36bf-477b-a3d5-971fdafee30e", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:28.668Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "c4500e9d-399f-458c-a467-a51372a8d5ae", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:34.131Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "da4a5ca0-be7f-4089-80d8-7619f202acb0", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:39.614Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "eaf5525a-e110-4c9f-8b61-02a4917aa69d", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:45.014Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "ada5dca1-641a-42da-81b4-9e1dc065fbf7", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:50.435Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "c7afb4c6-4883-46e1-a19e-5298a6ff596d", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:55.859Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "78c15b71-9aa9-4177-8d05-c3e749f8e2b3", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:58.962Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "5d0597ab-9978-4c1e-8f1d-5772c78cfe65", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.61.123.52:3306: connect: connection refused"} 2024-06-28T08:16:01.316Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "2df4e6bf-5a62-4717-b4df-cb678b6f0c8f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:03.807Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "63f49f7c-351e-431d-8a2f-a918ef8434be", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.61.123.52:3306: connect: connection refused"} 2024-06-28T08:16:06.726Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "17a68018-4c05-4b83-8e89-1c52696bdcb3", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:12.223Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "5ea06b85-351e-483e-9199-f361940912c2", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:17.616Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8ed1628f-bf9e-46e9-9909-cf3334fa386f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:23.195Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "7ac844ce-7e8c-404d-a239-3d27c4bdfb03", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:28.745Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "5a2efd87-2928-45b3-8b73-12a4f9f05244", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:34.283Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "f7c6e7e0-8fce-4c09-8048-d3a86dde929d", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:39.680Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "11a230a0-fa1d-489b-a8c6-18691cdb0c53", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:45.275Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "c7f3b89f-dfbf-494b-a5ee-2f07bc8c5007", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:50.670Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "701f968c-1180-4de1-b62e-407fee0b7e5a", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:56.074Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "b1c19188-ab77-47ac-b7e6-f7a050558477", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:01.530Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "dc91bf7c-7cd2-4c9b-b143-d9f197037263", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:06.909Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "51dc83c2-baba-454a-8e1a-5c873161d656", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:12.373Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "300f2662-4914-4990-9b33-d3a84749ecb9", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:17.875Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "a7b93130-96eb-4f8c-b97b-78ce614d9d7b", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:23.417Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "e3a6700d-e3c2-4c59-87bd-fb90f5e501ef", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:28.821Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "12c933c0-07cd-4980-914a-d1ca24ca3430", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:34.329Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "be46d821-ee8a-46b3-9a25-fed6737540d7", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:39.818Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "23eccbc3-e303-4e37-afc2-6080876e1f10", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:45.300Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "b4ec6dc3-902f-4e26-b96b-045f1963a993", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:50.675Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "5bb3aad7-69d6-4406-aa7c-2c0feed7b8aa", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:56.152Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "7f4ea99d-e4ed-4d43-9002-4d4221a1e47c", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:18:01.560Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "a043fc8e-542f-4210-a7fb-2f0b95848e48", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:18:07.011Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "a34821a4-a535-45cb-87e7-6221ba32c970", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:18:12.754Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "3654b10a-5e90-4961-81cb-45634dd30ae0", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:18:18.218Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "372d3baa-0763-4d91-ac9a-a0c77c7ade1f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:18:20.275Z INFO You can view xtrabackup log: 2024-06-28T08:18:44.190Z INFO Remove outdated replication channel {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "channel": "source_to_replica"} 2024-06-28T08:18:44.195Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "channel": "source_to_replica", "host": "10.61.115.122"} 2024-06-28T08:18:44.196Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "channel": "source_to_replica", "host": "10.61.122.184"} 2024-06-28T08:18:44.197Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "channel": "source_to_replica", "host": "10.61.125.225"} 2024-06-28T08:18:44.214Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "pod": "cross-site-replica-pxc-0"} 2024-06-28T08:18:44.224Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "pod": "cross-site-replica-pxc-2"} 2024-06-28T08:18:44.234Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "pod": "cross-site-replica-pxc-1"} 2024-06-28T08:18:47.078Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "f552157b-e21f-4418-b0c2-20f748066edf", "pod": "cross-site-source-pxc-1"} 2024-06-28T08:18:47.096Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "f552157b-e21f-4418-b0c2-20f748066edf", "pod": "cross-site-source-pxc-0"} 2024-06-28T08:18:47.117Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "f552157b-e21f-4418-b0c2-20f748066edf", "pod": "cross-site-source-pxc-2"} 2024-06-28T08:18:47.172Z INFO Replication pod has changed {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "f552157b-e21f-4418-b0c2-20f748066edf", "new replication pod": "cross-site-source-pxc-0"} {"controller": "pxcrestore-controller", "namespace": "cross-site-27746", "name": "backup-minio", "reconcileID": "6fe2e644-59c4-4d52-821f-e51151241789"} {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-20022", "name": "backup-minio", "reconcileID": "96262dcb-374a-43bd-bc4f-bce7b802699b"} /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 If everything is fine, you can cleanup the job: $ kubectl delete pxc-restore/backup-minio $ kubectl logs job/restore-job-backup-minio-cross-site-replica $ kubectl logs job/restore-job-backup-minio-cross-site-source [mysql] 2024/06/28 08:13:03 connection.go:49: read tcp 10.204.120.16:51882->10.204.121.19:3306: read: connection reset by peer sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n cross-site-27746 cross-site-source --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + kubectl patch pxc -n cross-site-replica-20022 cross-site-replica --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.2nJL8MbTdW ++ mktemp + local LAST_ERR=/tmp/tmp.0uGvh7kYMx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2nJL8MbTdW perconaxtradbcluster.pxc.percona.com "cross-site-source" deleted perconaxtradbcluster.pxc.percona.com "cross-site-replica" deleted + cat /tmp/tmp.0uGvh7kYMx + rm /tmp/tmp.2nJL8MbTdW /tmp/tmp.0uGvh7kYMx + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.D5dX0Lkc0J ++ mktemp + local LAST_ERR=/tmp/tmp.mTujz7gVlI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.D5dX0Lkc0J perconaxtradbclusterbackup.pxc.percona.com "backup-minio-source" deleted perconaxtradbclusterbackup.pxc.percona.com "backup-minio-replica" deleted + cat /tmp/tmp.mTujz7gVlI + rm /tmp/tmp.D5dX0Lkc0J /tmp/tmp.mTujz7gVlI + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.KIAOllmEYe ++ mktemp + local LAST_ERR=/tmp/tmp.ggTC37Y9SG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KIAOllmEYe perconaxtradbclusterrestore.pxc.percona.com "backup-minio" deleted perconaxtradbclusterrestore.pxc.percona.com "backup-minio" deleted + cat /tmp/tmp.ggTC37Y9SG + rm /tmp/tmp.KIAOllmEYe /tmp/tmp.ggTC37Y9SG + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.f39oJoradc ++ mktemp + local LAST_ERR=/tmp/tmp.Cn68sMkz7l + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.f39oJoradc validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.Cn68sMkz7l + rm /tmp/tmp.f39oJoradc /tmp/tmp.Cn68sMkz7l + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-27746 + rm -rf /tmp/tmp.Su6U2NCsGL + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp ++ mktemp + destroy cross-site-replica-20022 true + local namespace=cross-site-replica-20022 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.8BYIdWphrT + '[' true == false -o 1 == 1 ']' + local LAST_OUT=/tmp/tmp.a8hZRgA4A8 ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.QzDa3AulPZ + local exit_status=0 + local LAST_ERR=/tmp/tmp.Y8BLmrhqIR + local exit_status=0 + sort -u ++ get_operator_pod + tee /tmp/tmp.Su6U2NCsGL/operator.log ++ local label_prefix=app.kubernetes.io/ + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'get backup status: Job.batch' tee: /tmp/tmp.Su6U2NCsGL/operator.log: No such file or directory +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace cross-site-27746 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + grep -v level=info + grep -v 'the object has been modified' ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.UKhbjU6TDp +++ mktemp ++ local LAST_ERR=/tmp/tmp.7dl9RvCEiB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.UKhbjU6TDp ++ cat /tmp/tmp.7dl9RvCEiB ++ rm /tmp/tmp.UKhbjU6TDp /tmp/tmp.7dl9RvCEiB ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-6849457d9-c7dd7 ++ mktemp + local LAST_OUT=/tmp/tmp.YdsbkPcEXL ++ mktemp + local LAST_ERR=/tmp/tmp.fyka9dV5kP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-6849457d9-c7dd7 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YdsbkPcEXL + cat /tmp/tmp.fyka9dV5kP + rm /tmp/tmp.YdsbkPcEXL /tmp/tmp.fyka9dV5kP + return 0 2024-06-28T07:47:43.556Z INFO setup Manager starting up {"gitCommit": "0a840b68490b2f8881fb749474303f7fb8a1239d", "gitBranch": "PR-1740-0a840b68", "buildTime": "2024-06-28T06:58:02Z", "goVersion": "go1.22.4", "os": "linux", "arch": "amd64"} 2024-06-28T07:47:43.556Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.26.15-gke.1436000"} 2024-06-28T07:47:43.610Z INFO setup Registering Components. 2024-06-28T07:47:48.213Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2024-06-28T07:47:48.216Z INFO controller-runtime.metrics Starting metrics server 2024-06-28T07:47:48.216Z INFO setup Starting the Cmd. 2024-06-28T07:47:48.217Z INFO controller-runtime.certwatcher Starting certificate watcher 2024-06-28T07:47:48.217Z INFO controller-runtime.certwatcher Updated current TLS certificate 2024-06-28T07:47:48.217Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2024-06-28T07:47:48.217Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2024-06-28T07:47:48.217Z INFO controller-runtime.webhook Starting webhook server 2024-06-28T07:47:48.217Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2024-06-28T07:47:48.318Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2024-06-28T07:47:48.331Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2024-06-28T07:47:48.332Z DEBUG events percona-xtradb-cluster-operator-6849457d9-c7dd7_0c158c91-fe20-45c0-84ba-e150208f9bd1 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"5e2f0e36-640a-4099-aaba-dfc6afd138ef","apiVersion":"coordination.k8s.io/v1","resourceVersion":"24143"}, "reason": "LeaderElection"} 2024-06-28T07:47:48.332Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2024-06-28T07:47:48.332Z INFO Starting Controller {"controller": "pxc-controller"} 2024-06-28T07:47:48.332Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2024-06-28T07:47:48.332Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2024-06-28T07:47:48.332Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"} 2024-06-28T07:47:48.332Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2024-06-28T07:47:48.438Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2024-06-28T07:47:48.438Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2024-06-28T07:47:48.438Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2024-06-28T07:49:30.805Z INFO Set CR version {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "034d94d5-f097-451f-8f26-3f7571a461b5", "version": "1.15.0"} 2024-06-28T07:49:38.853Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 44cfafd5-0642-4002-8e83-62d612d126f9 2024-06-28T07:50:51.292Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9", "user": "operator"} 2024-06-28T07:50:51.334Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9", "user": "monitor"} 2024-06-28T07:50:51.440Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9"} 2024-06-28T07:50:51.482Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9"} 2024-06-28T07:50:51.519Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9", "user": "xtrabackup"} 2024-06-28T07:50:51.579Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9"} 2024-06-28T07:50:51.618Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9", "user": "replication"} 2024-06-28T07:50:53.723Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "a4f6edb5-ba1c-4933-ba3e-6c637f9ba3c9", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.61.123.52:3306: connect: connection refused"} 2024-06-28T07:53:20.264Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "b6326dbc-30dc-476f-bf63-c27bdab1b0e0", "user": "root"} 2024-06-28T07:53:20.618Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "b6326dbc-30dc-476f-bf63-c27bdab1b0e0", "new version": "8.0.36-28.1"} 2024-06-28T07:56:04.474Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "7acecdc5-6c20-48d5-8d24-6c66e8d9f849", "user": "replication"} 2024-06-28T07:56:04.525Z INFO Password updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "7acecdc5-6c20-48d5-8d24-6c66e8d9f849", "user": "replication"} 2024-06-28T07:56:04.538Z INFO MySQL init secret created {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "7acecdc5-6c20-48d5-8d24-6c66e8d9f849", "secret": "cross-site-source-mysql-init", "user": "replication"} 2024-06-28T07:56:04.551Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "7acecdc5-6c20-48d5-8d24-6c66e8d9f849", "user": "replication"} 2024-06-28T07:56:04.583Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "7acecdc5-6c20-48d5-8d24-6c66e8d9f849", "user": "replication"} 2024-06-28T07:56:50.120Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "cross-site-27746", "name": "backup-minio-source", "reconcileID": "d632aea5-281c-4e9f-af23-4866f86243e6", "Namespace": "cross-site-27746", "Name": "xb-backup-minio-source"} 2024-06-28T07:59:38.239Z INFO Set CR version {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "405fb30e-30f4-452b-b13d-11848f993bd7", "version": "1.15.0"} 2024-06-28T07:59:48.335Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 8bfe22cf-d3fa-48a5-b8c9-eec3fe48bab0 2024-06-28T08:01:06.085Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296", "user": "operator"} 2024-06-28T08:01:06.134Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296", "user": "monitor"} 2024-06-28T08:01:06.235Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296"} 2024-06-28T08:01:06.271Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296"} 2024-06-28T08:01:06.309Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296", "user": "xtrabackup"} 2024-06-28T08:01:06.366Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296"} 2024-06-28T08:01:06.403Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8132909a-2ab5-4e3d-abba-af9c1f368296", "user": "replication"} 2024-06-28T08:03:27.674Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "2117d8a7-1a3e-450d-b582-120b49b14d0f", "user": "root"} 2024-06-28T08:03:27.918Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "2117d8a7-1a3e-450d-b582-120b49b14d0f", "new version": "8.0.36-28.1"} 2024-06-28T08:05:56.781Z INFO backup restore request {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-20022", "name": "backup-minio", "reconcileID": "96262dcb-374a-43bd-bc4f-bce7b802699b"} 2024-06-28T08:05:56.834Z INFO stopping cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-20022", "name": "backup-minio", "reconcileID": "96262dcb-374a-43bd-bc4f-bce7b802699b", "cluster": "cross-site-replica"} 2024-06-28T08:06:42.990Z INFO starting restore {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-20022", "name": "backup-minio", "reconcileID": "96262dcb-374a-43bd-bc4f-bce7b802699b", "cluster": "cross-site-replica", "backup": ""} 2024-06-28T08:07:06.056Z INFO starting cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-20022", "name": "backup-minio", "reconcileID": "96262dcb-374a-43bd-bc4f-bce7b802699b", "cluster": "cross-site-replica"} 2024-06-28T08:10:11.235Z INFO You can view xtrabackup log: 2024-06-28T08:10:54.474Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "6083c2bc-125c-49eb-8d7f-9a98f0810298", "pod": "cross-site-replica-pxc-0"} 2024-06-28T08:10:54.495Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "6083c2bc-125c-49eb-8d7f-9a98f0810298", "pod": "cross-site-replica-pxc-1"} 2024-06-28T08:10:54.522Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "6083c2bc-125c-49eb-8d7f-9a98f0810298", "pod": "cross-site-replica-pxc-2"} 2024-06-28T08:10:54.571Z INFO Replication pod has changed {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "6083c2bc-125c-49eb-8d7f-9a98f0810298", "new replication pod": "cross-site-replica-pxc-0"} 2024-06-28T08:10:56.101Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "45c7917c-4e0a-44ed-a72a-15fe7d0b4185", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:00.497Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "d52d7e5b-ad8e-4874-a94e-9413dc4ee4c1", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:05.967Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "e269ccb1-51ce-4b5b-bb63-c6e83766691e", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:11.426Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "0f3fd240-3d33-4025-9827-d62c4e7c69c4", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:16.866Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "bae0579c-f9a8-4acc-ad01-872db849695d", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:23.084Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "2b03823d-3035-43ca-b615-91005cd63027", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:28.717Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "07891407-b2c8-4747-b9ed-f6ee2bb62a93", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:34.220Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "84c1c78e-9e8e-4e18-b939-09835b912107", "channel": "source_to_replica", "Last_IO_Error": "Error connecting to source 'replication@10.61.115.122:3306'. This was attempt 1/0, with a delay of 60 seconds between attempts. Message: Access denied for user 'replication'@'cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-repli' (using password: YES)"} 2024-06-28T08:11:39.540Z INFO Password changed, updating user {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "57160a3d-2fea-4eda-9c8f-612af742fbcd", "user": "replication"} 2024-06-28T08:11:39.576Z INFO Password updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "57160a3d-2fea-4eda-9c8f-612af742fbcd", "user": "replication"} 2024-06-28T08:11:39.586Z INFO MySQL init secret created {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "57160a3d-2fea-4eda-9c8f-612af742fbcd", "secret": "cross-site-replica-mysql-init", "user": "replication"} 2024-06-28T08:11:39.601Z INFO Internal secrets updated {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "57160a3d-2fea-4eda-9c8f-612af742fbcd", "user": "replication"} 2024-06-28T08:11:39.677Z INFO Old password discarded {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "57160a3d-2fea-4eda-9c8f-612af742fbcd", "user": "replication"} 2024-06-28T08:11:39.858Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "57160a3d-2fea-4eda-9c8f-612af742fbcd", "channel": "source_to_replica", "Last_IO_Error": ""} 2024-06-28T08:12:25.420Z INFO Created a new backup job {"controller": "pxcbackup-controller", "namespace": "cross-site-replica-20022", "name": "backup-minio-replica", "reconcileID": "c386b13a-ad73-4f58-ae87-35cf057a2589", "Namespace": "cross-site-replica-20022", "Name": "xb-backup-minio-replica"} 2024-06-28T08:12:59.582Z INFO backup restore request {"controller": "pxcrestore-controller", "namespace": "cross-site-27746", "name": "backup-minio", "reconcileID": "6fe2e644-59c4-4d52-821f-e51151241789"} 2024-06-28T08:12:59.715Z INFO stopping cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-27746", "name": "backup-minio", "reconcileID": "6fe2e644-59c4-4d52-821f-e51151241789", "cluster": "cross-site-source"} 2024-06-28T08:13:03.778Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "3469ee8b-7394-4379-9dde-df46d93fc466", "err": "failed to ensure cluster readonly status: connect to pod cross-site-replica-pxc-1: dial tcp 10.204.121.19:33062: connect: connection refused"} 2024-06-28T08:13:39.928Z INFO starting restore {"controller": "pxcrestore-controller", "namespace": "cross-site-27746", "name": "backup-minio", "reconcileID": "6fe2e644-59c4-4d52-821f-e51151241789", "cluster": "cross-site-source", "backup": ""} 2024-06-28T08:14:11.432Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "3ed77b41-1f4d-4065-8d0d-d17e6cf47af2", "err": "get primary pxc pod: failed to get proxy connection: dial tcp: lookup cross-site-replica-haproxy.cross-site-replica-20022: i/o timeout"} 2024-06-28T08:14:27.820Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "3087acb8-65cf-4f30-8e5e-696711d93d48", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:28.250Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "87cf6fa7-052a-4c26-afa2-008327c2d3c7", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:33.163Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "aec7abee-a2be-497a-9721-cf99e9da1d21", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:33.750Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "3d603911-a7d3-45cf-a6d0-f64c4a96acf7", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:38.509Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "1b1720f8-f61d-417f-ad4b-23ab2d0fa1e5", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:40.058Z INFO starting cluster {"controller": "pxcrestore-controller", "namespace": "cross-site-27746", "name": "backup-minio", "reconcileID": "6fe2e644-59c4-4d52-821f-e51151241789", "cluster": "cross-site-source"} 2024-06-28T08:14:44.126Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "d593e9a0-e6a8-4e9b-885d-d69d5e50d349", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:49.517Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "6938b603-199c-4601-b8ed-1e68174437cd", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:14:54.772Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "557bbc80-f625-47b4-9ecc-6e9ed5e50b0b", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:00.111Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "7ebf22ce-bb9d-4837-8e15-417665d2287c", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:05.512Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "9d77d5d2-36c1-4253-9f58-38a7d7693108", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:10.826Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "17400f69-c9aa-4e73-9314-d67a6ea75003", "pod": "cross-site-replica-pxc-1"} 2024-06-28T08:15:10.842Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "17400f69-c9aa-4e73-9314-d67a6ea75003", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:11.867Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "684d7998-baae-41aa-b30f-c72a55ae9cad", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:16.307Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "88ba3434-b850-45d5-b5e8-ff510372810c", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 2/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:21.909Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "6ee3484f-36bf-477b-a3d5-971fdafee30e", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:28.668Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "c4500e9d-399f-458c-a467-a51372a8d5ae", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:34.131Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "da4a5ca0-be7f-4089-80d8-7619f202acb0", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:39.614Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "eaf5525a-e110-4c9f-8b61-02a4917aa69d", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:45.014Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "ada5dca1-641a-42da-81b4-9e1dc065fbf7", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:50.435Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "c7afb4c6-4883-46e1-a19e-5298a6ff596d", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:55.859Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "78c15b71-9aa9-4177-8d05-c3e749f8e2b3", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:15:58.962Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "5d0597ab-9978-4c1e-8f1d-5772c78cfe65", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.61.123.52:3306: connect: connection refused"} 2024-06-28T08:16:01.316Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "2df4e6bf-5a62-4717-b4df-cb678b6f0c8f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:03.807Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "63f49f7c-351e-431d-8a2f-a918ef8434be", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.61.123.52:3306: connect: connection refused"} 2024-06-28T08:16:06.726Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "17a68018-4c05-4b83-8e89-1c52696bdcb3", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:12.223Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "5ea06b85-351e-483e-9199-f361940912c2", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:17.616Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "8ed1628f-bf9e-46e9-9909-cf3334fa386f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 3/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:23.195Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "7ac844ce-7e8c-404d-a239-3d27c4bdfb03", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:28.745Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "5a2efd87-2928-45b3-8b73-12a4f9f05244", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:34.283Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "f7c6e7e0-8fce-4c09-8048-d3a86dde929d", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:39.680Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "11a230a0-fa1d-489b-a8c6-18691cdb0c53", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:45.275Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "c7f3b89f-dfbf-494b-a5ee-2f07bc8c5007", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:50.670Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "701f968c-1180-4de1-b62e-407fee0b7e5a", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:16:56.074Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "b1c19188-ab77-47ac-b7e6-f7a050558477", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:01.530Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "dc91bf7c-7cd2-4c9b-b143-d9f197037263", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:06.909Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "51dc83c2-baba-454a-8e1a-5c873161d656", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:12.373Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "300f2662-4914-4990-9b33-d3a84749ecb9", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:17.875Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "a7b93130-96eb-4f8c-b97b-78ce614d9d7b", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 4/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:23.417Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "e3a6700d-e3c2-4c59-87bd-fb90f5e501ef", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:28.821Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "12c933c0-07cd-4980-914a-d1ca24ca3430", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:34.329Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "be46d821-ee8a-46b3-9a25-fed6737540d7", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:39.818Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "23eccbc3-e303-4e37-afc2-6080876e1f10", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:45.300Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "b4ec6dc3-902f-4e26-b96b-045f1963a993", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:50.675Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "5bb3aad7-69d6-4406-aa7c-2c0feed7b8aa", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:17:56.152Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "7f4ea99d-e4ed-4d43-9002-4d4221a1e47c", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:18:01.560Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "a043fc8e-542f-4210-a7fb-2f0b95848e48", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:18:07.011Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "a34821a4-a535-45cb-87e7-6221ba32c970", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:18:12.754Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "3654b10a-5e90-4961-81cb-45634dd30ae0", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:18:18.218Z INFO Replication for channel is not running. Please, check the replication status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "372d3baa-0763-4d91-ac9a-a0c77c7ade1f", "channel": "source_to_replica", "Last_IO_Error": "Error reconnecting to source 'replication@10.61.115.122:3306'. This was attempt 5/0, with a delay of 60 seconds between attempts. Message: Can't connect to MySQL server on '10.61.115.122:3306' (111)"} 2024-06-28T08:18:20.275Z INFO You can view xtrabackup log: 2024-06-28T08:18:44.190Z INFO Remove outdated replication channel {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "channel": "source_to_replica"} 2024-06-28T08:18:44.195Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "channel": "source_to_replica", "host": "10.61.115.122"} 2024-06-28T08:18:44.196Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "channel": "source_to_replica", "host": "10.61.122.184"} 2024-06-28T08:18:44.197Z DEBUG Remove outdated replication source {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "channel": "source_to_replica", "host": "10.61.125.225"} 2024-06-28T08:18:44.214Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "pod": "cross-site-replica-pxc-0"} 2024-06-28T08:18:44.224Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "pod": "cross-site-replica-pxc-2"} 2024-06-28T08:18:44.234Z INFO Primary is readonly. Disabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "798d50f8-7537-4747-a57c-936cc3bca686", "pod": "cross-site-replica-pxc-1"} 2024-06-28T08:18:47.078Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "f552157b-e21f-4418-b0c2-20f748066edf", "pod": "cross-site-source-pxc-1"} 2024-06-28T08:18:47.096Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "f552157b-e21f-4418-b0c2-20f748066edf", "pod": "cross-site-source-pxc-0"} 2024-06-28T08:18:47.117Z INFO Replica is not readonly. Enabling readonly mode {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "f552157b-e21f-4418-b0c2-20f748066edf", "pod": "cross-site-source-pxc-2"} 2024-06-28T08:18:47.172Z INFO Replication pod has changed {"controller": "pxc-controller", "namespace": "cross-site-27746", "name": "cross-site-source", "reconcileID": "f552157b-e21f-4418-b0c2-20f748066edf", "new replication pod": "cross-site-source-pxc-0"} 2024-06-28T08:19:37.833Z ERROR Update status {"controller": "pxc-controller", "namespace": "cross-site-replica-20022", "name": "cross-site-replica", "reconcileID": "a23f0f12-775c-47bd-8dad-8a578fa92d96", "error": "write status: PerconaXtraDBCluster.pxc.percona.com \"cross-site-replica\" not found", "errorVerbose": "PerconaXtraDBCluster.pxc.percona.com \"cross-site-replica\" not found\nwrite status\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).writeStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:157\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updateStatus\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/status.go:140\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:248\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:455\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} {"controller": "pxcrestore-controller", "namespace": "cross-site-27746", "name": "backup-minio", "reconcileID": "6fe2e644-59c4-4d52-821f-e51151241789"} {"controller": "pxcrestore-controller", "namespace": "cross-site-replica-20022", "name": "backup-minio", "reconcileID": "96262dcb-374a-43bd-bc4f-bce7b802699b"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile.func1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:114 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:311 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:250 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:455 If everything is fine, you can cleanup the job: $ kubectl delete pxc-restore/backup-minio $ kubectl logs job/restore-job-backup-minio-cross-site-replica $ kubectl logs job/restore-job-backup-minio-cross-site-source [mysql] 2024/06/28 08:13:03 connection.go:49: read tcp 10.204.120.16:51882->10.204.121.19:3306: read: connection reset by peer sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 + : + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.uZLOBxgbYT ++ mktemp + local LAST_ERR=/tmp/tmp.yWKmRxYneA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uZLOBxgbYT No resources found + cat /tmp/tmp.yWKmRxYneA + rm /tmp/tmp.uZLOBxgbYT /tmp/tmp.yWKmRxYneA + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.o42qLzQL5i ++ mktemp + local LAST_ERR=/tmp/tmp.G6w4mxUvuV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.o42qLzQL5i No resources found + cat /tmp/tmp.G6w4mxUvuV + rm /tmp/tmp.o42qLzQL5i /tmp/tmp.G6w4mxUvuV + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.ZsjJsYJdWy ++ mktemp + local LAST_ERR=/tmp/tmp.41Ge8ORuKd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZsjJsYJdWy No resources found + cat /tmp/tmp.41Ge8ORuKd + rm /tmp/tmp.ZsjJsYJdWy /tmp/tmp.41Ge8ORuKd + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.kygoRo3pc6 ++ mktemp + local LAST_ERR=/tmp/tmp.SE1MH5Yjz1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.kygoRo3pc6 + cat /tmp/tmp.SE1MH5Yjz1 Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "percona-xtradbcluster-webhook" not found + rm /tmp/tmp.kygoRo3pc6 /tmp/tmp.SE1MH5Yjz1 + return 1 + : + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8BYIdWphrT namespace "pxc-operator" force deleted + cat /tmp/tmp.QzDa3AulPZ Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.8BYIdWphrT /tmp/tmp.QzDa3AulPZ + return 0 + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + rm -rf /tmp/tmp.Su6U2NCsGL + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-replica-20022 + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- ++ mktemp + local LAST_OUT=/tmp/tmp.dSq0JaFGCv ++ mktemp + local LAST_OUT=/tmp/tmp.O9bObuushg ++ mktemp + local LAST_ERR=/tmp/tmp.DfLnulZS0K + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.rSEIDM7mJF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace cross-site-replica-20022 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.dSq0JaFGCv + cat /tmp/tmp.rSEIDM7mJF Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. Error from server (NotFound): namespaces "pxc-operator" not found + rm /tmp/tmp.dSq0JaFGCv /tmp/tmp.rSEIDM7mJF + return 1