Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/logs/cross-site-8-0.log WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.29) and server (1.25) exceeds the supported minor version skew of +/-1 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + main + local source_cluster=cross-site-source + local replica_cluster=cross-site-replica + desc 'Create source cluster' + set +o xtrace ----------------------------------------------------------------------------------- Create source cluster ----------------------------------------------------------------------------------- + create_infra cross-site-9668 + local ns=cross-site-9668 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n cross-site-11355 cross-site-source --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + kubectl patch pxc -n cross-site-replica-12113 cross-site-replica --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.Z8wOkd8zv2 ++ mktemp + local LAST_ERR=/tmp/tmp.Q6ys2R95zG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Z8wOkd8zv2 perconaxtradbcluster.pxc.percona.com "cross-site-source" deleted perconaxtradbcluster.pxc.percona.com "cross-site-replica" deleted + cat /tmp/tmp.Q6ys2R95zG + rm /tmp/tmp.Z8wOkd8zv2 /tmp/tmp.Q6ys2R95zG + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.wLnRoQBPsC ++ mktemp + local LAST_ERR=/tmp/tmp.KSrNwVyOG0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wLnRoQBPsC perconaxtradbclusterbackup.pxc.percona.com "backup-minio-source" deleted perconaxtradbclusterbackup.pxc.percona.com "backup-minio-replica" deleted + cat /tmp/tmp.KSrNwVyOG0 + rm /tmp/tmp.wLnRoQBPsC /tmp/tmp.KSrNwVyOG0 + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.hVnOqqYzqU ++ mktemp + local LAST_ERR=/tmp/tmp.3z4LQgcuor + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hVnOqqYzqU perconaxtradbclusterrestore.pxc.percona.com "backup-minio" deleted perconaxtradbclusterrestore.pxc.percona.com "backup-minio" deleted + cat /tmp/tmp.3z4LQgcuor + rm /tmp/tmp.hVnOqqYzqU /tmp/tmp.3z4LQgcuor + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' + '[' -n '' ']' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrole ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.1qjSkhxFXz ++ mktemp + local LAST_ERR=/tmp/tmp.OsQaO2dqWa + local exit_status=0 ++ seq 0 2 + xargs kubectl delete ns ++ mktemp + awk '{print$1}' + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_OUT=/tmp/tmp.XktW6B0EAZ ++ mktemp + local LAST_ERR=/tmp/tmp.gFaplngIcw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1qjSkhxFXz + cat /tmp/tmp.OsQaO2dqWa + rm /tmp/tmp.1qjSkhxFXz /tmp/tmp.OsQaO2dqWa + return 0 namespace "cross-site-11355" deleted namespace "cross-site-replica-12113" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XktW6B0EAZ namespace "pxc-operator" deleted + cat /tmp/tmp.gFaplngIcw + rm /tmp/tmp.XktW6B0EAZ /tmp/tmp.gFaplngIcw + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'namespace/pxc-operator - ' namespace/pxc-operator - + set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.VwdbynwLHr ++ mktemp + local LAST_ERR=/tmp/tmp.t9m5M77PZX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VwdbynwLHr namespace/pxc-operator created + cat /tmp/tmp.t9m5M77PZX + rm /tmp/tmp.VwdbynwLHr /tmp/tmp.t9m5M77PZX + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.morfNsF6aY +++ mktemp ++ local LAST_ERR=/tmp/tmp.uD0mv3hWIN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.morfNsF6aY ++ cat /tmp/tmp.uD0mv3hWIN ++ rm /tmp/tmp.morfNsF6aY /tmp/tmp.uD0mv3hWIN ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.6cVeO03DIC ++ mktemp + local LAST_ERR=/tmp/tmp.HhK5NYsMcK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6cVeO03DIC Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3" modified. + cat /tmp/tmp.HhK5NYsMcK + rm /tmp/tmp.6cVeO03DIC /tmp/tmp.HhK5NYsMcK + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.290HPndcS4 ++ mktemp + local LAST_ERR=/tmp/tmp.ST2q6xpJrP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.290HPndcS4 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.ST2q6xpJrP + rm /tmp/tmp.290HPndcS4 /tmp/tmp.ST2q6xpJrP + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.4kLBXLylxn ++ mktemp + local LAST_ERR=/tmp/tmp.sXWS0wLr0Z + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4kLBXLylxn clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.sXWS0wLr0Z + rm /tmp/tmp.4kLBXLylxn /tmp/tmp.sXWS0wLr0Z + return 0 + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1522-1ed4605b^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/deploy/cw-operator.yaml + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - ++ mktemp + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - + local LAST_OUT=/tmp/tmp.Q3drR6OVmt ++ mktemp + local LAST_ERR=/tmp/tmp.73NtMHtrMY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Q3drR6OVmt deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.73NtMHtrMY + rm /tmp/tmp.Q3drR6OVmt /tmp/tmp.73NtMHtrMY + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.Ln53aFtUd2 ++ mktemp + local LAST_ERR=/tmp/tmp.yEhf3OWNZg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Ln53aFtUd2 pod/percona-xtradb-cluster-operator-6dbd475d79-mk78v condition met + cat /tmp/tmp.yEhf3OWNZg + rm /tmp/tmp.Ln53aFtUd2 /tmp/tmp.yEhf3OWNZg + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.wehOpioE7N +++ mktemp ++ local LAST_ERR=/tmp/tmp.xhx9k4dr3m ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wehOpioE7N ++ cat /tmp/tmp.xhx9k4dr3m ++ rm /tmp/tmp.wehOpioE7N /tmp/tmp.xhx9k4dr3m ++ return 0 + wait_pod percona-xtradb-cluster-operator-6dbd475d79-mk78v 480 pxc-operator + local pod=percona-xtradb-cluster-operator-6dbd475d79-mk78v + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-6dbd475d79-mk78v ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-6dbd475d79-mk78v condition met percona-xtradb-cluster-operator-6dbd475d79-mk78v.Ok + sleep 3 + create_namespace cross-site-9668 + local namespace=cross-site-9668 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' + '[' -n '' ']' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-9668' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-9668++ mktemp ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-9668 ++ mktemp + local LAST_OUT=/tmp/tmp.PO36IInox4 + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + local LAST_OUT=/tmp/tmp.PNfwq7L3NU + awk '{print$1}' ++ mktemp + xargs kubectl delete ns + local LAST_ERR=/tmp/tmp.EPD1UNRJPy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns ++ mktemp + local LAST_ERR=/tmp/tmp.iY2x3V3yWA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-9668 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-9668 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PO36IInox4 + cat /tmp/tmp.EPD1UNRJPy + rm /tmp/tmp.PO36IInox4 /tmp/tmp.EPD1UNRJPy + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-9668 Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.PNfwq7L3NU + cat /tmp/tmp.iY2x3V3yWA Error from server (NotFound): namespaces "cross-site-9668" not found + rm /tmp/tmp.PNfwq7L3NU /tmp/tmp.iY2x3V3yWA + return 1 + : + wait_for_delete namespace/cross-site-9668 + local res=namespace/cross-site-9668 + echo -n 'namespace/cross-site-9668 - ' namespace/cross-site-9668 - + set +o xtrace Error from server (NotFound): namespaces "cross-site-9668" not found + desc 'create namespace cross-site-9668' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-9668 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-9668 ++ mktemp + local LAST_OUT=/tmp/tmp.GOxxcUX1Ol ++ mktemp + local LAST_ERR=/tmp/tmp.NH4XSV7AVC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cross-site-9668 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GOxxcUX1Ol namespace/cross-site-9668 created + cat /tmp/tmp.NH4XSV7AVC + rm /tmp/tmp.GOxxcUX1Ol /tmp/tmp.NH4XSV7AVC + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pd34BQbOBm +++ mktemp ++ local LAST_ERR=/tmp/tmp.wJ0fe4KwZP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Pd34BQbOBm ++ cat /tmp/tmp.wJ0fe4KwZP ++ rm /tmp/tmp.Pd34BQbOBm /tmp/tmp.wJ0fe4KwZP ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3 --namespace=cross-site-9668 ++ mktemp + local LAST_OUT=/tmp/tmp.pk7lvzU0cF ++ mktemp + local LAST_ERR=/tmp/tmp.OqVrupu79S + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3 --namespace=cross-site-9668 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pk7lvzU0cF Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3" modified. + cat /tmp/tmp.OqVrupu79S + rm /tmp/tmp.pk7lvzU0cF /tmp/tmp.OqVrupu79S + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.e8sOrT0RY3 ++ mktemp + local LAST_ERR=/tmp/tmp.JuvfuQq1ea + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.e8sOrT0RY3 secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.JuvfuQq1ea + rm /tmp/tmp.e8sOrT0RY3 /tmp/tmp.JuvfuQq1ea + return 0 + start_minio + deploy_helm cross-site-9668 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "chaos-mesh" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret= + local endpoint=http://minio-service:9000 + minio_args=(--version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set securityContext.enabled=false --set persistence.size=2G) + local minio_args + [[ -n '' ]] + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set securityContext.enabled=false --set persistence.size=2G minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set securityContext.enabled=false --set persistence.size=2G minio/minio NAME: minio-service LAST DEPLOYED: Fri Dec 29 11:07:56 2023 NAMESPACE: cross-site-9668 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.cross-site-9668.svc.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace cross-site-9668 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace cross-site-9668 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace cross-site-9668 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace cross-site-9668 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local + sleep 30 ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.c5rfqk55Qw +++ mktemp ++ local LAST_ERR=/tmp/tmp.u8SEmQ6LIW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.c5rfqk55Qw ++ cat /tmp/tmp.u8SEmQ6LIW ++ rm /tmp/tmp.c5rfqk55Qw /tmp/tmp.u8SEmQ6LIW ++ return 0 + MINIO_POD=minio-service-75dd45bdcd-khxg4 + wait_pod minio-service-75dd45bdcd-khxg4 + local pod=minio-service-75dd45bdcd-khxg4 + local max_retry=480 + local ns= ++ echo minio-service-75dd45bdcd-khxg4 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minio-service-75dd45bdcd-khxg4 condition met minio-service-75dd45bdcd-khxg4.Ok + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.iRvEPIwMh5 ++ mktemp + local LAST_ERR=/tmp/tmp.NmX4rDVWq9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iRvEPIwMh5 make_bucket: operator-testing pod "aws-cli" deleted + cat /tmp/tmp.NmX4rDVWq9 + rm /tmp/tmp.iRvEPIwMh5 /tmp/tmp.NmX4rDVWq9 + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.XJbYjVuAKH ++ mktemp + local LAST_ERR=/tmp/tmp.0yCznDIAId + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XJbYjVuAKH secret/minio-secret unchanged secret/aws-s3-secret unchanged secret/gcp-cs-secret unchanged secret/azure-secret unchanged + cat /tmp/tmp.0yCznDIAId + rm /tmp/tmp.XJbYjVuAKH /tmp/tmp.0yCznDIAId + return 0 + spinup_pxc cross-site-source /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/cross-site-source.yml + local cluster=cross-site-source + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/cross-site-source.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xS5CisgQMg ++ mktemp + local LAST_ERR=/tmp/tmp.K3X8cXatGw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xS5CisgQMg secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.K3X8cXatGw + rm /tmp/tmp.xS5CisgQMg /tmp/tmp.K3X8cXatGw + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1522-1ed4605b#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:k8spxc-1313-8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_OUT=/tmp/tmp.NJx32e2Fxm + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-9668~ + local LAST_ERR=/tmp/tmp.6GmEwzbAE0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NJx32e2Fxm deployment.apps/pxc-client created + cat /tmp/tmp.6GmEwzbAE0 + rm /tmp/tmp.NJx32e2Fxm /tmp/tmp.6GmEwzbAE0 + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/cross-site-source.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/cross-site-source.yml + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:k8spxc-1313-8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1522-1ed4605b#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-9668~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/cross-site-source.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.G6lftMW9dF ++ mktemp + local LAST_ERR=/tmp/tmp.Bef34vVuAY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.G6lftMW9dF perconaxtradbcluster.pxc.percona.com/cross-site-source created + cat /tmp/tmp.Bef34vVuAY + rm /tmp/tmp.G6lftMW9dF /tmp/tmp.Bef34vVuAY + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy cross-site-source ++ local target_cluster=cross-site-source +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ToVlqF8rXP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VQOlGZREf9 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ToVlqF8rXP +++ cat /tmp/tmp.VQOlGZREf9 +++ rm /tmp/tmp.ToVlqF8rXP /tmp/tmp.VQOlGZREf9 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo cross-site-source-haproxy ++ return + local proxy=cross-site-source-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-9668 ++ mktemp + local LAST_OUT=/tmp/tmp.ZM3mT2cbFJ ++ mktemp + local LAST_ERR=/tmp/tmp.gL5dqdgrgQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-9668 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-9668 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-9668 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.ZM3mT2cbFJ + cat /tmp/tmp.gL5dqdgrgQ error: no matching resources found + rm /tmp/tmp.ZM3mT2cbFJ /tmp/tmp.gL5dqdgrgQ + return 1 + true + wait_for_running cross-site-source-haproxy 1 + local name=cross-site-source-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-haproxy-0 480 + local pod=cross-site-source-haproxy-0 + local max_retry=480 + local ns= ++ echo cross-site-source-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace Error from server (NotFound): pods "cross-site-source-haproxy-0" not found cross-site-source-haproxy-0..................................Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running cross-site-source-pxc 3 + local name=cross-site-source-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-0 480 + local pod=cross-site-source-pxc-0 + local max_retry=480 + local ns= ++ echo cross-site-source-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/cross-site-source-pxc-0 condition met cross-site-source-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-1 480 + local pod=cross-site-source-pxc-1 + local max_retry=480 + local ns= ++ echo cross-site-source-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-source-pxc-1 condition met cross-site-source-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-2 480 + local pod=cross-site-source-pxc-2 + local max_retry=480 + local ns= ++ echo cross-site-source-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-source-pxc-2 condition met cross-site-source-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h cross-site-source-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h cross-site-source-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.p10LcwvWvq +++ mktemp ++ local LAST_ERR=/tmp/tmp.KVFdHeX3an ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.p10LcwvWvq ++ cat /tmp/tmp.KVFdHeX3an ++ rm /tmp/tmp.p10LcwvWvq /tmp/tmp.KVFdHeX3an ++ return 0 + client_pod=pxc-client-599db5b9bc-mm5w6 + wait_pod pxc-client-599db5b9bc-mm5w6 + local pod=pxc-client-599db5b9bc-mm5w6 + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-mm5w6 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-mm5w6 condition met pxc-client-599db5b9bc-mm5w6.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h cross-site-source-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h cross-site-source-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WUvLXHNFNI +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZpJjbZ3PuF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WUvLXHNFNI ++ cat /tmp/tmp.ZpJjbZ3PuF ++ rm /tmp/tmp.WUvLXHNFNI /tmp/tmp.ZpJjbZ3PuF ++ return 0 + client_pod=pxc-client-599db5b9bc-mm5w6 + wait_pod pxc-client-599db5b9bc-mm5w6 + local pod=pxc-client-599db5b9bc-mm5w6 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-599db5b9bc-mm5w6 + local container= + set +o xtrace pod/pxc-client-599db5b9bc-mm5w6 condition met pxc-client-599db5b9bc-mm5w6.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.35zaXYiOMG +++ mktemp ++ local LAST_ERR=/tmp/tmp.mjzstdtXl2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.35zaXYiOMG ++ cat /tmp/tmp.mjzstdtXl2 ++ rm /tmp/tmp.35zaXYiOMG /tmp/tmp.mjzstdtXl2 ++ return 0 + client_pod=pxc-client-599db5b9bc-mm5w6 + wait_pod pxc-client-599db5b9bc-mm5w6 + local pod=pxc-client-599db5b9bc-mm5w6 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-599db5b9bc-mm5w6 + local container= + set +o xtrace pod/pxc-client-599db5b9bc-mm5w6 condition met pxc-client-599db5b9bc-mm5w6.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.nxIQyhbhUS/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.nxIQyhbhUS/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ra0EGOkI8k +++ mktemp ++ local LAST_ERR=/tmp/tmp.KYoDQSXfR0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ra0EGOkI8k ++ cat /tmp/tmp.KYoDQSXfR0 ++ rm /tmp/tmp.ra0EGOkI8k /tmp/tmp.KYoDQSXfR0 ++ return 0 + client_pod=pxc-client-599db5b9bc-mm5w6 + wait_pod pxc-client-599db5b9bc-mm5w6 + local pod=pxc-client-599db5b9bc-mm5w6 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-599db5b9bc-mm5w6 + local container= + set +o xtrace pod/pxc-client-599db5b9bc-mm5w6 condition met pxc-client-599db5b9bc-mm5w6.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.nxIQyhbhUS/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.nxIQyhbhUS/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.h28tdc7ZmD +++ mktemp ++ local LAST_ERR=/tmp/tmp.zf0KNoPYgD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.h28tdc7ZmD ++ cat /tmp/tmp.zf0KNoPYgD ++ rm /tmp/tmp.h28tdc7ZmD /tmp/tmp.zf0KNoPYgD ++ return 0 + client_pod=pxc-client-599db5b9bc-mm5w6 + wait_pod pxc-client-599db5b9bc-mm5w6 + local pod=pxc-client-599db5b9bc-mm5w6 + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-mm5w6 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-mm5w6 condition met pxc-client-599db5b9bc-mm5w6.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.nxIQyhbhUS/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.nxIQyhbhUS/select-1.sql ++ is_keyring_plugin_in_use cross-site-source ++ local cluster=cross-site-source ++ kubectl_bin exec -it cross-site-source-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OYleJK3N70 +++ mktemp ++ local LAST_ERR=/tmp/tmp.b5cKhF6YTL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it cross-site-source-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OYleJK3N70 ++ cat /tmp/tmp.b5cKhF6YTL Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.OYleJK3N70 /tmp/tmp.b5cKhF6YTL ++ return 0 + '[' '' ']' + sleep 60 + desc 'get main cluster services endpoints' + set +o xtrace ----------------------------------------------------------------------------------- get main cluster services endpoints ----------------------------------------------------------------------------------- ++ get_service_ip cross-site-source-pxc-0 ++ local service=cross-site-source-pxc-0 ++ grep -q NotFound ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.YkFTzt6OKl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zztlPkLzOz +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.YkFTzt6OKl +++ cat /tmp/tmp.zztlPkLzOz +++ rm /tmp/tmp.YkFTzt6OKl /tmp/tmp.zztlPkLzOz +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qb05MjD67P +++ mktemp ++ local LAST_ERR=/tmp/tmp.3dwAmZbQ41 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qb05MjD67P ++ cat /tmp/tmp.3dwAmZbQ41 ++ rm /tmp/tmp.qb05MjD67P /tmp/tmp.3dwAmZbQ41 ++ return 0 ++ return + source_endpoint0=10.7.138.253 ++ get_service_ip cross-site-source-pxc-1 ++ local service=cross-site-source-pxc-1 ++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zpAqmPJQDo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.u6qW35EvDz +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.zpAqmPJQDo +++ cat /tmp/tmp.u6qW35EvDz +++ rm /tmp/tmp.zpAqmPJQDo /tmp/tmp.u6qW35EvDz +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5AHcAiXGlw +++ mktemp ++ local LAST_ERR=/tmp/tmp.mMas1ZlfKh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5AHcAiXGlw ++ cat /tmp/tmp.mMas1ZlfKh ++ rm /tmp/tmp.5AHcAiXGlw /tmp/tmp.mMas1ZlfKh ++ return 0 ++ return + source_endpoint1=10.7.130.151 ++ get_service_ip cross-site-source-pxc-2 ++ local service=cross-site-source-pxc-2 ++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CMfA8EtTQE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UEL06TgwMT +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.CMfA8EtTQE +++ cat /tmp/tmp.UEL06TgwMT +++ rm /tmp/tmp.CMfA8EtTQE /tmp/tmp.UEL06TgwMT +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QUt1pejMTK +++ mktemp ++ local LAST_ERR=/tmp/tmp.FYBNaVDqR8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QUt1pejMTK ++ cat /tmp/tmp.FYBNaVDqR8 ++ rm /tmp/tmp.QUt1pejMTK /tmp/tmp.FYBNaVDqR8 ++ return 0 ++ return + source_endpoint2=10.7.136.76 ++ run_mysql 'SELECT @@hostname hostname;' '-h cross-site-source-haproxy -p33062 -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h cross-site-source-haproxy -p33062 -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.AYMfAdKr8O ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hfPQFaSZo5 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.AYMfAdKr8O +++ cat /tmp/tmp.hfPQFaSZo5 +++ rm /tmp/tmp.AYMfAdKr8O /tmp/tmp.hfPQFaSZo5 +++ return 0 ++ client_pod=pxc-client-599db5b9bc-mm5w6 ++ wait_pod pxc-client-599db5b9bc-mm5w6 ++ local pod=pxc-client-599db5b9bc-mm5w6 ++ local max_retry=480 ++ local ns= +++ echo pxc-client-599db5b9bc-mm5w6 +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-599db5b9bc-mm5w6 condition met pxc-client-599db5b9bc-mm5w6.Ok ++ set +o xtrace + source_primary=cross-site-source-pxc-0 ++ get_service_ip cross-site-source-pxc-0 ++ local service=cross-site-source-pxc-0 ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.nbpitg9z6n ++++ mktemp +++ local LAST_ERR=/tmp/tmp.AMQYrSs1Pj +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.nbpitg9z6n +++ cat /tmp/tmp.AMQYrSs1Pj +++ rm /tmp/tmp.nbpitg9z6n /tmp/tmp.AMQYrSs1Pj +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cBxx3ch4Mj +++ mktemp ++ local LAST_ERR=/tmp/tmp.Tdh5uRpq2n ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cBxx3ch4Mj ++ cat /tmp/tmp.Tdh5uRpq2n ++ rm /tmp/tmp.cBxx3ch4Mj /tmp/tmp.Tdh5uRpq2n ++ return 0 ++ return + source_primary_endpoint=10.7.138.253 + desc 'patch source cluster with replicationChannels settings' + set +o xtrace ----------------------------------------------------------------------------------- patch source cluster with replicationChannels settings ----------------------------------------------------------------------------------- + kubectl_bin patch pxc cross-site-source --type=merge --patch '{"spec": {"pxc":{"replicationChannels": [{"name":"source_to_replica", "isSource": true}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.FkrxKgyQvZ ++ mktemp + local LAST_ERR=/tmp/tmp.Dg3uJqew0l + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-source --type=merge --patch '{"spec": {"pxc":{"replicationChannels": [{"name":"source_to_replica", "isSource": true}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FkrxKgyQvZ perconaxtradbcluster.pxc.percona.com/cross-site-source patched + cat /tmp/tmp.Dg3uJqew0l + rm /tmp/tmp.FkrxKgyQvZ /tmp/tmp.Dg3uJqew0l + return 0 + desc 'patch main cluster secrets with replication user' + set +o xtrace ----------------------------------------------------------------------------------- patch main cluster secrets with replication user ----------------------------------------------------------------------------------- ++ echo -n new_password ++ base64 + kubectl_bin patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.JzsMj4Cf9F ++ mktemp + local LAST_ERR=/tmp/tmp.zUQ5LzizOy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JzsMj4Cf9F secret/my-cluster-secrets patched + cat /tmp/tmp.zUQ5LzizOy + rm /tmp/tmp.JzsMj4Cf9F /tmp/tmp.zUQ5LzizOy + return 0 + sleep 15 + wait_cluster_consistency cross-site-source 3 2 + local cluster_name=cross-site-source + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.apX1yiXqlg +++ mktemp ++ local LAST_ERR=/tmp/tmp.72tIgrLYVL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.apX1yiXqlg ++ cat /tmp/tmp.72tIgrLYVL ++ rm /tmp/tmp.apX1yiXqlg /tmp/tmp.72tIgrLYVL ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kUxZktJ6Dq +++ mktemp ++ local LAST_ERR=/tmp/tmp.R2bNJFLvBB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kUxZktJ6Dq ++ cat /tmp/tmp.R2bNJFLvBB ++ rm /tmp/tmp.kUxZktJ6Dq /tmp/tmp.R2bNJFLvBB ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.2CoynaVqj5 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ocDuK9KK8R +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.2CoynaVqj5 +++++ cat /tmp/tmp.ocDuK9KK8R +++++ rm /tmp/tmp.2CoynaVqj5 /tmp/tmp.ocDuK9KK8R +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-haproxy ++++ return +++ local cluster_proxy=cross-site-source-haproxy +++ echo haproxy ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hIPgmmzO6k +++ mktemp ++ local LAST_ERR=/tmp/tmp.rRWndSnxs4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hIPgmmzO6k ++ cat /tmp/tmp.rRWndSnxs4 ++ rm /tmp/tmp.hIPgmmzO6k /tmp/tmp.rRWndSnxs4 ++ return 0 + [[ 2 == \2 ]] + desc 'write data to source cluster' + set +o xtrace ----------------------------------------------------------------------------------- write data to source cluster ----------------------------------------------------------------------------------- + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testSourceReplica (id int PRIMARY KEY);' '-h 10.7.138.253 -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testSourceReplica (id int PRIMARY KEY);' + local 'uri=-h 10.7.138.253 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FOdeZFBp28 +++ mktemp ++ local LAST_ERR=/tmp/tmp.i0PYRCKlTA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FOdeZFBp28 ++ cat /tmp/tmp.i0PYRCKlTA ++ rm /tmp/tmp.FOdeZFBp28 /tmp/tmp.i0PYRCKlTA ++ return 0 + client_pod=pxc-client-599db5b9bc-mm5w6 + wait_pod pxc-client-599db5b9bc-mm5w6 + local pod=pxc-client-599db5b9bc-mm5w6 + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-mm5w6 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-mm5w6 condition met pxc-client-599db5b9bc-mm5w6.Ok + set +o xtrace + run_mysql 'INSERT myApp.testSourceReplica (id) VALUES (100700)' '-h 10.7.138.253 -uroot -proot_password' + local 'command=INSERT myApp.testSourceReplica (id) VALUES (100700)' + local 'uri=-h 10.7.138.253 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.InDHa9qnQI +++ mktemp ++ local LAST_ERR=/tmp/tmp.l8QAMZSkMm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.InDHa9qnQI ++ cat /tmp/tmp.l8QAMZSkMm ++ rm /tmp/tmp.InDHa9qnQI /tmp/tmp.l8QAMZSkMm ++ return 0 + client_pod=pxc-client-599db5b9bc-mm5w6 + wait_pod pxc-client-599db5b9bc-mm5w6 + local pod=pxc-client-599db5b9bc-mm5w6 + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-mm5w6 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-mm5w6 condition met pxc-client-599db5b9bc-mm5w6.Ok + set +o xtrace + kubectl_bin get secrets cross-site-source-ssl-internal -o yaml + yq 'del(.metadata)' - + yq '.metadata={"name": "cross-site-replica-ssl-internal"}' - ++ mktemp + local LAST_OUT=/tmp/tmp.ZosvpSsLJg ++ mktemp + local LAST_ERR=/tmp/tmp.3VD47iv1mr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get secrets cross-site-source-ssl-internal -o yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZosvpSsLJg + cat /tmp/tmp.3VD47iv1mr + rm /tmp/tmp.ZosvpSsLJg /tmp/tmp.3VD47iv1mr + return 0 + desc 'take backup of source cluster' + set +o xtrace ----------------------------------------------------------------------------------- take backup of source cluster ----------------------------------------------------------------------------------- + run_backup cross-site-source backup-minio-source + local cluster=cross-site-source + local backup1=backup-minio-source + desc 'make backup backup-minio-source' + set +o xtrace ----------------------------------------------------------------------------------- make backup backup-minio-source ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/backup-minio-source.yml ++ mktemp + local LAST_OUT=/tmp/tmp.MHQ5gHI3XR ++ mktemp + local LAST_ERR=/tmp/tmp.Gy4vgfR0JL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/backup-minio-source.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MHQ5gHI3XR perconaxtradbclusterbackup.pxc.percona.com/backup-minio-source created + cat /tmp/tmp.Gy4vgfR0JL + rm /tmp/tmp.MHQ5gHI3XR /tmp/tmp.Gy4vgfR0JL + return 0 + wait_backup backup-minio-source + local backup=backup-minio-source + local status=Succeeded + set +o xtrace backup-minio-source................Succeeded + desc 'create replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- create replica cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-replica-27495 0 + local namespace=cross-site-replica-27495 + local skip_clean_namespace=0 + [[ 1 == 1 ]] + [[ -z 0 ]] + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-replica-27495' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-replica-27495 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-replica-27495 ++ mktemp + local LAST_OUT=/tmp/tmp.MoNxKwg6yR ++ mktemp + local LAST_ERR=/tmp/tmp.eDAUVMN0LR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-replica-27495 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-replica-27495 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-replica-27495 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.MoNxKwg6yR + cat /tmp/tmp.eDAUVMN0LR Error from server (NotFound): namespaces "cross-site-replica-27495" not found + rm /tmp/tmp.MoNxKwg6yR /tmp/tmp.eDAUVMN0LR + return 1 + : + wait_for_delete namespace/cross-site-replica-27495 + local res=namespace/cross-site-replica-27495 + echo -n 'namespace/cross-site-replica-27495 - ' namespace/cross-site-replica-27495 - + set +o xtrace Error from server (NotFound): namespaces "cross-site-replica-27495" not found + desc 'create namespace cross-site-replica-27495' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-replica-27495 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-replica-27495 ++ mktemp + local LAST_OUT=/tmp/tmp.9qqMNan9z2 ++ mktemp + local LAST_ERR=/tmp/tmp.J0ZooZpcEf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cross-site-replica-27495 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9qqMNan9z2 namespace/cross-site-replica-27495 created + cat /tmp/tmp.J0ZooZpcEf + rm /tmp/tmp.9qqMNan9z2 /tmp/tmp.J0ZooZpcEf + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.nfS3ZsAnpv +++ mktemp ++ local LAST_ERR=/tmp/tmp.G7nLlOaW8O ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.nfS3ZsAnpv ++ cat /tmp/tmp.G7nLlOaW8O ++ rm /tmp/tmp.nfS3ZsAnpv /tmp/tmp.G7nLlOaW8O ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3 --namespace=cross-site-replica-27495 ++ mktemp + local LAST_OUT=/tmp/tmp.3uOjw4y0go ++ mktemp + local LAST_ERR=/tmp/tmp.7nizNfFHlL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3 --namespace=cross-site-replica-27495 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3uOjw4y0go Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3" modified. + cat /tmp/tmp.7nizNfFHlL + rm /tmp/tmp.3uOjw4y0go /tmp/tmp.7nizNfFHlL + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.WlqOp8XcLk ++ mktemp + local LAST_ERR=/tmp/tmp.mL70sc0G8e + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WlqOp8XcLk customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.mL70sc0G8e + rm /tmp/tmp.WlqOp8XcLk /tmp/tmp.mL70sc0G8e + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.6NIybXPDal ++ mktemp + local LAST_ERR=/tmp/tmp.ChALyrFqqY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6NIybXPDal clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.ChALyrFqqY + rm /tmp/tmp.6NIybXPDal /tmp/tmp.ChALyrFqqY + return 0 + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1522-1ed4605b^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/deploy/cw-operator.yaml + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - ++ mktemp + local LAST_OUT=/tmp/tmp.SPEUcZNquv ++ mktemp + local LAST_ERR=/tmp/tmp.dU3kiXj9mx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SPEUcZNquv deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.dU3kiXj9mx + rm /tmp/tmp.SPEUcZNquv /tmp/tmp.dU3kiXj9mx + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.IuPpM4hCO1 ++ mktemp + local LAST_ERR=/tmp/tmp.57d5sxrRLx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.IuPpM4hCO1 + cat /tmp/tmp.57d5sxrRLx error: timed out waiting for the condition on pods/percona-xtradb-cluster-operator-6dbd475d79-z5xq9 + rm /tmp/tmp.IuPpM4hCO1 /tmp/tmp.57d5sxrRLx + return 1 + true ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.e2dpcUodon +++ mktemp ++ local LAST_ERR=/tmp/tmp.oSQxtPhofs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.e2dpcUodon ++ cat /tmp/tmp.oSQxtPhofs ++ rm /tmp/tmp.e2dpcUodon /tmp/tmp.oSQxtPhofs ++ return 0 + wait_pod percona-xtradb-cluster-operator-6dbd475d79-mk78v 480 pxc-operator + local pod=percona-xtradb-cluster-operator-6dbd475d79-mk78v + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-6dbd475d79-mk78v ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-6dbd475d79-mk78v condition met percona-xtradb-cluster-operator-6dbd475d79-mk78v.Ok + sleep 3 + kubectl_bin apply -f /tmp/tmp.nxIQyhbhUS/replica-ssl-internal.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.USrGL56FkI ++ mktemp + local LAST_ERR=/tmp/tmp.fJUPf54ulr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /tmp/tmp.nxIQyhbhUS/replica-ssl-internal.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.USrGL56FkI secret/cross-site-replica-ssl-internal created + cat /tmp/tmp.fJUPf54ulr + rm /tmp/tmp.USrGL56FkI /tmp/tmp.fJUPf54ulr + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.DMxwnqiyKD ++ mktemp + local LAST_ERR=/tmp/tmp.49YOqp7QeW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DMxwnqiyKD secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.49YOqp7QeW + rm /tmp/tmp.DMxwnqiyKD /tmp/tmp.49YOqp7QeW + return 0 + spinup_pxc cross-site-replica /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/cross-site-replica.yml + local cluster=cross-site-replica + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/cross-site-replica.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.yl5drlsBZN ++ mktemp + local LAST_ERR=/tmp/tmp.5nED7ldJMk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yl5drlsBZN secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.5nED7ldJMk + rm /tmp/tmp.yl5drlsBZN /tmp/tmp.5nED7ldJMk + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1522-1ed4605b#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-9668~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:k8spxc-1313-8.0-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.KrzSL0r7Wq ++ mktemp + local LAST_ERR=/tmp/tmp.jl7faRN2Gi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KrzSL0r7Wq deployment.apps/pxc-client created + cat /tmp/tmp.jl7faRN2Gi + rm /tmp/tmp.KrzSL0r7Wq /tmp/tmp.jl7faRN2Gi + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/cross-site-replica.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/cross-site-replica.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/cross-site-replica.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.C9ramwU1zm + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1522-1ed4605b#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:k8spxc-1313-8.0-backup#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_ERR=/tmp/tmp.mUsTl2o88K + local exit_status=0 + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ seq 0 2 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-9668~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.C9ramwU1zm perconaxtradbcluster.pxc.percona.com/cross-site-replica created + cat /tmp/tmp.mUsTl2o88K + rm /tmp/tmp.C9ramwU1zm /tmp/tmp.mUsTl2o88K + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy cross-site-replica ++ local target_cluster=cross-site-replica +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NUDXIEACb5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.prQwunDZpr +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.NUDXIEACb5 +++ cat /tmp/tmp.prQwunDZpr +++ rm /tmp/tmp.NUDXIEACb5 /tmp/tmp.prQwunDZpr +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo cross-site-replica-haproxy ++ return + local proxy=cross-site-replica-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-9668 ++ mktemp + local LAST_OUT=/tmp/tmp.C0IGjl9hpO ++ mktemp + local LAST_ERR=/tmp/tmp.369B0JpuBW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-9668 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-9668 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n cross-site-9668 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.C0IGjl9hpO + cat /tmp/tmp.369B0JpuBW error: no matching resources found + rm /tmp/tmp.C0IGjl9hpO /tmp/tmp.369B0JpuBW + return 1 + true + wait_for_running cross-site-replica-haproxy 1 + local name=cross-site-replica-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-haproxy-0 480 + local pod=cross-site-replica-haproxy-0 + local max_retry=480 + local ns= ++ echo cross-site-replica-haproxy-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/cross-site-replica-haproxy-0 condition met cross-site-replica-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running cross-site-replica-pxc 3 + local name=cross-site-replica-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-0 480 + local pod=cross-site-replica-pxc-0 + local max_retry=480 + local ns= ++ echo cross-site-replica-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-replica-pxc-0 condition met cross-site-replica-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-1 480 + local pod=cross-site-replica-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo cross-site-replica-pxc-1 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/cross-site-replica-pxc-1 condition met cross-site-replica-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-2 480 + local pod=cross-site-replica-pxc-2 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo cross-site-replica-pxc-2 + local container=pxc + set +o xtrace pod/cross-site-replica-pxc-2 condition met cross-site-replica-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h cross-site-replica-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h cross-site-replica-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cbJx8g24Vf +++ mktemp ++ local LAST_ERR=/tmp/tmp.FAY0TtF4tq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.cbJx8g24Vf ++ cat /tmp/tmp.FAY0TtF4tq ++ rm /tmp/tmp.cbJx8g24Vf /tmp/tmp.FAY0TtF4tq ++ return 0 + client_pod=pxc-client-599db5b9bc-5wznj + wait_pod pxc-client-599db5b9bc-5wznj + local pod=pxc-client-599db5b9bc-5wznj + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-5wznj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-5wznj condition met pxc-client-599db5b9bc-5wznj.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h cross-site-replica-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h cross-site-replica-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.35fBxeJKcu +++ mktemp ++ local LAST_ERR=/tmp/tmp.BVAH0ZQTN6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.35fBxeJKcu ++ cat /tmp/tmp.BVAH0ZQTN6 ++ rm /tmp/tmp.35fBxeJKcu /tmp/tmp.BVAH0ZQTN6 ++ return 0 + client_pod=pxc-client-599db5b9bc-5wznj + wait_pod pxc-client-599db5b9bc-5wznj + local pod=pxc-client-599db5b9bc-5wznj + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-5wznj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-5wznj condition met pxc-client-599db5b9bc-5wznj.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.y8buAxairi +++ mktemp ++ local LAST_ERR=/tmp/tmp.bTrB6duep6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.y8buAxairi ++ cat /tmp/tmp.bTrB6duep6 ++ rm /tmp/tmp.y8buAxairi /tmp/tmp.bTrB6duep6 ++ return 0 + client_pod=pxc-client-599db5b9bc-5wznj + wait_pod pxc-client-599db5b9bc-5wznj + local pod=pxc-client-599db5b9bc-5wznj + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-599db5b9bc-5wznj + local container= + set +o xtrace pod/pxc-client-599db5b9bc-5wznj condition met pxc-client-599db5b9bc-5wznj.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.nxIQyhbhUS/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.nxIQyhbhUS/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wthXR15vyS +++ mktemp ++ local LAST_ERR=/tmp/tmp.e3SQCMXtX2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wthXR15vyS ++ cat /tmp/tmp.e3SQCMXtX2 ++ rm /tmp/tmp.wthXR15vyS /tmp/tmp.e3SQCMXtX2 ++ return 0 + client_pod=pxc-client-599db5b9bc-5wznj + wait_pod pxc-client-599db5b9bc-5wznj + local pod=pxc-client-599db5b9bc-5wznj + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-5wznj ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-5wznj condition met pxc-client-599db5b9bc-5wznj.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.nxIQyhbhUS/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.nxIQyhbhUS/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3TliVDWnrk +++ mktemp ++ local LAST_ERR=/tmp/tmp.WjdvrWfP9N ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3TliVDWnrk ++ cat /tmp/tmp.WjdvrWfP9N ++ rm /tmp/tmp.3TliVDWnrk /tmp/tmp.WjdvrWfP9N ++ return 0 + client_pod=pxc-client-599db5b9bc-5wznj + wait_pod pxc-client-599db5b9bc-5wznj + local pod=pxc-client-599db5b9bc-5wznj + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-599db5b9bc-5wznj + local container= + set +o xtrace pod/pxc-client-599db5b9bc-5wznj condition met pxc-client-599db5b9bc-5wznj.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.nxIQyhbhUS/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.nxIQyhbhUS/select-1.sql ++ is_keyring_plugin_in_use cross-site-replica ++ local cluster=cross-site-replica ++ kubectl_bin exec -it cross-site-replica-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.J7fQdwSKoH +++ mktemp ++ local LAST_ERR=/tmp/tmp.VD5uJTlJo5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it cross-site-replica-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.J7fQdwSKoH ++ cat /tmp/tmp.VD5uJTlJo5 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.J7fQdwSKoH /tmp/tmp.VD5uJTlJo5 ++ return 0 + '[' '' ']' + sleep 60 + desc 'restore backup from source cluster' + set +o xtrace ----------------------------------------------------------------------------------- restore backup from source cluster ----------------------------------------------------------------------------------- ++ kubectl_bin get -n cross-site-9668 pxc-backup backup-minio-source -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TWEpbxEZoT +++ mktemp ++ local LAST_ERR=/tmp/tmp.sBcmd1bd5x ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get -n cross-site-9668 pxc-backup backup-minio-source -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.TWEpbxEZoT ++ cat /tmp/tmp.sBcmd1bd5x ++ rm /tmp/tmp.TWEpbxEZoT /tmp/tmp.sBcmd1bd5x ++ return 0 + destination=s3://operator-testing/cross-site-source-2023-12-29-11:16:30-full + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/restore-backup-minio.yml + /usr/bin/sed -e s~#destination~s3://operator-testing/cross-site-source-2023-12-29-11:16:30-full~ + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-9668~ + /usr/bin/sed -e s~#cluster~cross-site-replica~ + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.WCRJi5hZc9 ++ mktemp + local LAST_ERR=/tmp/tmp.vxMDMg1SDR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.WCRJi5hZc9 perconaxtradbclusterrestore.pxc.percona.com/backup-minio created + cat /tmp/tmp.vxMDMg1SDR + rm /tmp/tmp.WCRJi5hZc9 /tmp/tmp.vxMDMg1SDR + return 0 + wait_cluster_consistency cross-site-replica 3 2 + local cluster_name=cross-site-replica + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WQNEYpqDrx +++ mktemp ++ local LAST_ERR=/tmp/tmp.9W2S4xyWiV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WQNEYpqDrx ++ cat /tmp/tmp.9W2S4xyWiV ++ rm /tmp/tmp.WQNEYpqDrx /tmp/tmp.9W2S4xyWiV ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qHGHeIJksw +++ mktemp ++ local LAST_ERR=/tmp/tmp.4JCpUUCxkU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qHGHeIJksw ++ cat /tmp/tmp.4JCpUUCxkU ++ rm /tmp/tmp.qHGHeIJksw /tmp/tmp.4JCpUUCxkU ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.o0IhnVCNeM +++ mktemp ++ local LAST_ERR=/tmp/tmp.uvo0DMpG2n ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.o0IhnVCNeM ++ cat /tmp/tmp.uvo0DMpG2n ++ rm /tmp/tmp.o0IhnVCNeM /tmp/tmp.uvo0DMpG2n ++ return 0 + [[ paused == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.chJqMKVP9Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.NDFE6SFYPU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.chJqMKVP9Q ++ cat /tmp/tmp.NDFE6SFYPU ++ rm /tmp/tmp.chJqMKVP9Q /tmp/tmp.NDFE6SFYPU ++ return 0 + [[ paused == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RGccAiQdIh +++ mktemp ++ local LAST_ERR=/tmp/tmp.kATE7ptobB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RGccAiQdIh ++ cat /tmp/tmp.kATE7ptobB ++ rm /tmp/tmp.RGccAiQdIh /tmp/tmp.kATE7ptobB ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tcLAr6fFYq +++ mktemp ++ local LAST_ERR=/tmp/tmp.436LTrHiaX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tcLAr6fFYq ++ cat /tmp/tmp.436LTrHiaX ++ rm /tmp/tmp.tcLAr6fFYq /tmp/tmp.436LTrHiaX ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mvsbtgQw9L +++ mktemp ++ local LAST_ERR=/tmp/tmp.AbgpHVgE4Q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.mvsbtgQw9L ++ cat /tmp/tmp.AbgpHVgE4Q ++ rm /tmp/tmp.mvsbtgQw9L /tmp/tmp.AbgpHVgE4Q ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NPUKCrbYDx +++ mktemp ++ local LAST_ERR=/tmp/tmp.OOS772I77d ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NPUKCrbYDx ++ cat /tmp/tmp.OOS772I77d ++ rm /tmp/tmp.NPUKCrbYDx /tmp/tmp.OOS772I77d ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.S8cDXCE3jg +++ mktemp ++ local LAST_ERR=/tmp/tmp.JhBs42cizV ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.S8cDXCE3jg ++ cat /tmp/tmp.JhBs42cizV ++ rm /tmp/tmp.S8cDXCE3jg /tmp/tmp.JhBs42cizV ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.P6yxkLSevF +++ mktemp ++ local LAST_ERR=/tmp/tmp.PvDGXMbaSg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.P6yxkLSevF ++ cat /tmp/tmp.PvDGXMbaSg ++ rm /tmp/tmp.P6yxkLSevF /tmp/tmp.PvDGXMbaSg ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w62Z3ddxxf +++ mktemp ++ local LAST_ERR=/tmp/tmp.GGrwE3Fy2R ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.w62Z3ddxxf ++ cat /tmp/tmp.GGrwE3Fy2R ++ rm /tmp/tmp.w62Z3ddxxf /tmp/tmp.GGrwE3Fy2R ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EdX198M7LS +++ mktemp ++ local LAST_ERR=/tmp/tmp.8mT1aVKnIP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EdX198M7LS ++ cat /tmp/tmp.8mT1aVKnIP ++ rm /tmp/tmp.EdX198M7LS /tmp/tmp.8mT1aVKnIP ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dRObOfONJS +++ mktemp ++ local LAST_ERR=/tmp/tmp.fsNkVwI0AK ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dRObOfONJS ++ cat /tmp/tmp.fsNkVwI0AK ++ rm /tmp/tmp.dRObOfONJS /tmp/tmp.fsNkVwI0AK ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EpDRg49G9F +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZURDL701tW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EpDRg49G9F ++ cat /tmp/tmp.ZURDL701tW ++ rm /tmp/tmp.EpDRg49G9F /tmp/tmp.ZURDL701tW ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.QWzAMfxPnv ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.cHzjXNvyFk +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.QWzAMfxPnv +++++ cat /tmp/tmp.cHzjXNvyFk +++++ rm /tmp/tmp.QWzAMfxPnv /tmp/tmp.cHzjXNvyFk +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-haproxy ++++ return +++ local cluster_proxy=cross-site-replica-haproxy +++ echo haproxy ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aphbj9LpUJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.WN0nOkTisH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aphbj9LpUJ ++ cat /tmp/tmp.WN0nOkTisH ++ rm /tmp/tmp.aphbj9LpUJ /tmp/tmp.WN0nOkTisH ++ return 0 + [[ 2 == \2 ]] + desc 'get replica cluster services endpoints' + set +o xtrace ----------------------------------------------------------------------------------- get replica cluster services endpoints ----------------------------------------------------------------------------------- ++ get_service_ip cross-site-replica-pxc-0 ++ local service=cross-site-replica-pxc-0 ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Wn8tC9uGtV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0NqPOUlz4a +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Wn8tC9uGtV +++ cat /tmp/tmp.0NqPOUlz4a +++ rm /tmp/tmp.Wn8tC9uGtV /tmp/tmp.0NqPOUlz4a +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bcGkXUdkjp +++ mktemp ++ local LAST_ERR=/tmp/tmp.5XdZHYTulb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bcGkXUdkjp ++ cat /tmp/tmp.5XdZHYTulb ++ rm /tmp/tmp.bcGkXUdkjp /tmp/tmp.5XdZHYTulb ++ return 0 ++ return + replica_endpoint0=10.7.129.40 ++ get_service_ip cross-site-replica-pxc-1 ++ local service=cross-site-replica-pxc-1 ++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.62CcaV3Z25 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wcK7W69E9I +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.62CcaV3Z25 +++ cat /tmp/tmp.wcK7W69E9I +++ rm /tmp/tmp.62CcaV3Z25 /tmp/tmp.wcK7W69E9I +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hLlLi7tzZM +++ mktemp ++ local LAST_ERR=/tmp/tmp.KOzFij1EqC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hLlLi7tzZM ++ cat /tmp/tmp.KOzFij1EqC ++ rm /tmp/tmp.hLlLi7tzZM /tmp/tmp.KOzFij1EqC ++ return 0 ++ return + replica_endpoint1=10.7.140.7 ++ get_service_ip cross-site-replica-pxc-2 ++ local service=cross-site-replica-pxc-2 ++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PsQQkNkmDg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.N5l8ZfWx8m +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.PsQQkNkmDg +++ cat /tmp/tmp.N5l8ZfWx8m +++ rm /tmp/tmp.PsQQkNkmDg /tmp/tmp.N5l8ZfWx8m +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WKb0hTw3bB +++ mktemp ++ local LAST_ERR=/tmp/tmp.r86hT1Ptgy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WKb0hTw3bB ++ cat /tmp/tmp.r86hT1Ptgy ++ rm /tmp/tmp.WKb0hTw3bB /tmp/tmp.r86hT1Ptgy ++ return 0 ++ return + replica_endpoint2=10.7.143.112 ++ run_mysql 'SELECT @@hostname hostname;' '-h cross-site-replica-haproxy -p33062 -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h cross-site-replica-haproxy -p33062 -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ZPHRxeJfUp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YeYHzb9vMS +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.ZPHRxeJfUp +++ cat /tmp/tmp.YeYHzb9vMS +++ rm /tmp/tmp.ZPHRxeJfUp /tmp/tmp.YeYHzb9vMS +++ return 0 ++ client_pod=pxc-client-599db5b9bc-5wznj ++ wait_pod pxc-client-599db5b9bc-5wznj ++ local pod=pxc-client-599db5b9bc-5wznj ++ local max_retry=480 ++ local ns= +++ echo pxc-client-599db5b9bc-5wznj +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-599db5b9bc-5wznj condition met pxc-client-599db5b9bc-5wznj.Ok ++ set +o xtrace + replica_primary=cross-site-replica-pxc-0 ++ get_service_ip cross-site-replica-pxc-0 ++ local service=cross-site-replica-pxc-0 ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CrZgBByVQ9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3lBav4UcXk +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.CrZgBByVQ9 +++ cat /tmp/tmp.3lBav4UcXk +++ rm /tmp/tmp.CrZgBByVQ9 /tmp/tmp.3lBav4UcXk +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0D9kG1aqpp +++ mktemp ++ local LAST_ERR=/tmp/tmp.20aSTqOPBb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0D9kG1aqpp ++ cat /tmp/tmp.20aSTqOPBb ++ rm /tmp/tmp.0D9kG1aqpp /tmp/tmp.20aSTqOPBb ++ return 0 ++ return + replica_primary_endpoint=10.7.129.40 + run_mysql 'DELETE FROM myApp.myApp WHERE id=100500' '-h 10.7.129.40 -uroot -proot_password' + local 'command=DELETE FROM myApp.myApp WHERE id=100500' + local 'uri=-h 10.7.129.40 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0IaQcbBSsY +++ mktemp ++ local LAST_ERR=/tmp/tmp.gsoe3TxLhz ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.0IaQcbBSsY ++ cat /tmp/tmp.gsoe3TxLhz ++ rm /tmp/tmp.0IaQcbBSsY /tmp/tmp.gsoe3TxLhz ++ return 0 + client_pod=pxc-client-599db5b9bc-5wznj + wait_pod pxc-client-599db5b9bc-5wznj + local pod=pxc-client-599db5b9bc-5wznj + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-5wznj ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-5wznj condition met pxc-client-599db5b9bc-5wznj.Ok + set +o xtrace + desc 'patch replica cluster with replicationChannels settings' + set +o xtrace ----------------------------------------------------------------------------------- patch replica cluster with replicationChannels settings ----------------------------------------------------------------------------------- + kubectl_bin patch pxc cross-site-replica --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "10.7.138.253", "port": 3306, "weight": 100},{"host": "10.7.130.151", "port": 3306, "weight": 100},{"host": "10.7.136.76", "port": 3306, "weight": 100}]}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.SQcQ7mBvcO ++ mktemp + local LAST_ERR=/tmp/tmp.8Jo3kPVTUd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-replica --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "10.7.138.253", "port": 3306, "weight": 100},{"host": "10.7.130.151", "port": 3306, "weight": 100},{"host": "10.7.136.76", "port": 3306, "weight": 100}]}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SQcQ7mBvcO perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + cat /tmp/tmp.8Jo3kPVTUd + rm /tmp/tmp.SQcQ7mBvcO /tmp/tmp.8Jo3kPVTUd + return 0 + sleep 40 + desc 'patch replica cluster secrets with replication user' + set +o xtrace ----------------------------------------------------------------------------------- patch replica cluster secrets with replication user ----------------------------------------------------------------------------------- ++ echo -n new_password ++ base64 + kubectl_bin patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.vBB7F2a1kX ++ mktemp + local LAST_ERR=/tmp/tmp.PJWwoOuxBH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vBB7F2a1kX secret/my-cluster-secrets patched + cat /tmp/tmp.PJWwoOuxBH + rm /tmp/tmp.vBB7F2a1kX /tmp/tmp.PJWwoOuxBH + return 0 + sleep 15 + wait_cluster_consistency cross-site-replica 3 2 + local cluster_name=cross-site-replica + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kDOOsZFC9V +++ mktemp ++ local LAST_ERR=/tmp/tmp.fmC2U5FCvi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kDOOsZFC9V ++ cat /tmp/tmp.fmC2U5FCvi ++ rm /tmp/tmp.kDOOsZFC9V /tmp/tmp.fmC2U5FCvi ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.a6ysSz1ZBH +++ mktemp ++ local LAST_ERR=/tmp/tmp.Su9h0W5g4p ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.a6ysSz1ZBH ++ cat /tmp/tmp.Su9h0W5g4p ++ rm /tmp/tmp.a6ysSz1ZBH /tmp/tmp.Su9h0W5g4p ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.VDo2M2laTu ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Z4vbwiLjId +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.VDo2M2laTu +++++ cat /tmp/tmp.Z4vbwiLjId +++++ rm /tmp/tmp.VDo2M2laTu /tmp/tmp.Z4vbwiLjId +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-haproxy ++++ return +++ local cluster_proxy=cross-site-replica-haproxy +++ echo haproxy ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ML9ZwHUvdK +++ mktemp ++ local LAST_ERR=/tmp/tmp.hx6e7Cn63z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ML9ZwHUvdK ++ cat /tmp/tmp.hx6e7Cn63z ++ rm /tmp/tmp.ML9ZwHUvdK /tmp/tmp.hx6e7Cn63z ++ return 0 + [[ 2 == \2 ]] + desc 'Check replication works between source -> replica' + set +o xtrace ----------------------------------------------------------------------------------- Check replication works between source -> replica ----------------------------------------------------------------------------------- + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 10.7.129.40 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.7.129.40 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-2-80.sql ']' + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 10.7.129.40 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.7.129.40 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NexCZKloKG +++ mktemp ++ local LAST_ERR=/tmp/tmp.w3K474wZ0a ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NexCZKloKG ++ cat /tmp/tmp.w3K474wZ0a ++ rm /tmp/tmp.NexCZKloKG /tmp/tmp.w3K474wZ0a ++ return 0 + client_pod=pxc-client-599db5b9bc-5wznj + wait_pod pxc-client-599db5b9bc-5wznj + local pod=pxc-client-599db5b9bc-5wznj + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-5wznj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-5wznj condition met pxc-client-599db5b9bc-5wznj.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.nxIQyhbhUS/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-2.sql /tmp/tmp.nxIQyhbhUS/select-2.sql + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 10.7.140.7 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.7.140.7 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-2-80.sql ']' + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 10.7.140.7 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.7.140.7 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wEImoDLr9h +++ mktemp ++ local LAST_ERR=/tmp/tmp.6uUqgep0yj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wEImoDLr9h ++ cat /tmp/tmp.6uUqgep0yj ++ rm /tmp/tmp.wEImoDLr9h /tmp/tmp.6uUqgep0yj ++ return 0 + client_pod=pxc-client-599db5b9bc-5wznj + wait_pod pxc-client-599db5b9bc-5wznj + local pod=pxc-client-599db5b9bc-5wznj + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-5wznj ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-5wznj condition met pxc-client-599db5b9bc-5wznj.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.nxIQyhbhUS/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-2.sql /tmp/tmp.nxIQyhbhUS/select-2.sql + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 10.7.143.112 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.7.143.112 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-2-80.sql ']' + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 10.7.143.112 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.7.143.112 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lZoXdEvwMG +++ mktemp ++ local LAST_ERR=/tmp/tmp.wIdAmesWJr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lZoXdEvwMG ++ cat /tmp/tmp.wIdAmesWJr ++ rm /tmp/tmp.lZoXdEvwMG /tmp/tmp.wIdAmesWJr ++ return 0 + client_pod=pxc-client-599db5b9bc-5wznj + wait_pod pxc-client-599db5b9bc-5wznj + local pod=pxc-client-599db5b9bc-5wznj + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-5wznj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-5wznj condition met pxc-client-599db5b9bc-5wznj.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.nxIQyhbhUS/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-2.sql /tmp/tmp.nxIQyhbhUS/select-2.sql + run_backup cross-site-replica backup-minio-replica + local cluster=cross-site-replica + local backup1=backup-minio-replica + desc 'make backup backup-minio-replica' + set +o xtrace ----------------------------------------------------------------------------------- make backup backup-minio-replica ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/backup-minio-replica.yml ++ mktemp + local LAST_OUT=/tmp/tmp.TfUHlP4xmZ ++ mktemp + local LAST_ERR=/tmp/tmp.n2Y4s4ctHF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/backup-minio-replica.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.TfUHlP4xmZ perconaxtradbclusterbackup.pxc.percona.com/backup-minio-replica created + cat /tmp/tmp.n2Y4s4ctHF + rm /tmp/tmp.TfUHlP4xmZ /tmp/tmp.n2Y4s4ctHF + return 0 + wait_backup backup-minio-replica + local backup=backup-minio-replica + local status=Succeeded + set +o xtrace backup-minio-replica.............Succeeded + desc 'Switch clusters over' + set +o xtrace ----------------------------------------------------------------------------------- Switch clusters over ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Au0hbLBGjU +++ mktemp ++ local LAST_ERR=/tmp/tmp.TAR4kyV07q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Au0hbLBGjU ++ cat /tmp/tmp.TAR4kyV07q ++ rm /tmp/tmp.Au0hbLBGjU /tmp/tmp.TAR4kyV07q ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3 --namespace=cross-site-9668 ++ mktemp + local LAST_OUT=/tmp/tmp.ApS39lSJu6 ++ mktemp + local LAST_ERR=/tmp/tmp.DyCzSoGOMs + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3 --namespace=cross-site-9668 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ApS39lSJu6 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3" modified. + cat /tmp/tmp.DyCzSoGOMs + rm /tmp/tmp.ApS39lSJu6 /tmp/tmp.DyCzSoGOMs + return 0 + desc 'rebuild source cluster' + set +o xtrace ----------------------------------------------------------------------------------- rebuild source cluster ----------------------------------------------------------------------------------- ++ kubectl_bin get -n cross-site-replica-27495 pxc-backup backup-minio-replica -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.149RCjcRyj +++ mktemp ++ local LAST_ERR=/tmp/tmp.LTEueco6EA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get -n cross-site-replica-27495 pxc-backup backup-minio-replica -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.149RCjcRyj ++ cat /tmp/tmp.LTEueco6EA ++ rm /tmp/tmp.149RCjcRyj /tmp/tmp.LTEueco6EA ++ return 0 + destination=s3://operator-testing/cross-site-replica-2023-12-29-11:31:53-full + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/conf/restore-backup-minio.yml + /usr/bin/sed -e s~#destination~s3://operator-testing/cross-site-replica-2023-12-29-11:31:53-full~ + kubectl_bin apply -f - + /usr/bin/sed -e s~#cluster~cross-site-source~ + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-9668~ ++ mktemp + local LAST_OUT=/tmp/tmp.nCrqiOH3bU ++ mktemp + local LAST_ERR=/tmp/tmp.Qoexq3SApM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.nCrqiOH3bU perconaxtradbclusterrestore.pxc.percona.com/backup-minio created + cat /tmp/tmp.Qoexq3SApM + rm /tmp/tmp.nCrqiOH3bU /tmp/tmp.Qoexq3SApM + return 0 + wait_cluster_consistency cross-site-source 3 2 + local cluster_name=cross-site-source + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + sleep 7 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xPuqryLbtH +++ mktemp ++ local LAST_ERR=/tmp/tmp.QDkou8SW9U ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xPuqryLbtH ++ cat /tmp/tmp.QDkou8SW9U ++ rm /tmp/tmp.xPuqryLbtH /tmp/tmp.QDkou8SW9U ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.chBlS23Q86 +++ mktemp ++ local LAST_ERR=/tmp/tmp.33ECEtM5xF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.chBlS23Q86 ++ cat /tmp/tmp.33ECEtM5xF ++ rm /tmp/tmp.chBlS23Q86 /tmp/tmp.33ECEtM5xF ++ return 0 + [[ stopping == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jl1xpxZxgh +++ mktemp ++ local LAST_ERR=/tmp/tmp.t6hpZY3auj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.jl1xpxZxgh ++ cat /tmp/tmp.t6hpZY3auj ++ rm /tmp/tmp.jl1xpxZxgh /tmp/tmp.t6hpZY3auj ++ return 0 + [[ paused == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DfqLmG5fzQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.IiX6R5sNbX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DfqLmG5fzQ ++ cat /tmp/tmp.IiX6R5sNbX ++ rm /tmp/tmp.DfqLmG5fzQ /tmp/tmp.IiX6R5sNbX ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xWJfyG7kAl +++ mktemp ++ local LAST_ERR=/tmp/tmp.GJZ6wjLeUB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.xWJfyG7kAl ++ cat /tmp/tmp.GJZ6wjLeUB ++ rm /tmp/tmp.xWJfyG7kAl /tmp/tmp.GJZ6wjLeUB ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ra5374FP5v +++ mktemp ++ local LAST_ERR=/tmp/tmp.OvNnfspbdA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ra5374FP5v ++ cat /tmp/tmp.OvNnfspbdA ++ rm /tmp/tmp.Ra5374FP5v /tmp/tmp.OvNnfspbdA ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5lIzVqsna5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.MFn7Sf3kSs ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5lIzVqsna5 ++ cat /tmp/tmp.MFn7Sf3kSs ++ rm /tmp/tmp.5lIzVqsna5 /tmp/tmp.MFn7Sf3kSs ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YNiNVrwwiu +++ mktemp ++ local LAST_ERR=/tmp/tmp.WR1fgqhVfv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YNiNVrwwiu ++ cat /tmp/tmp.WR1fgqhVfv ++ rm /tmp/tmp.YNiNVrwwiu /tmp/tmp.WR1fgqhVfv ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9YuPnmG9PI +++ mktemp ++ local LAST_ERR=/tmp/tmp.zknKM1xl3D ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.9YuPnmG9PI ++ cat /tmp/tmp.zknKM1xl3D ++ rm /tmp/tmp.9YuPnmG9PI /tmp/tmp.zknKM1xl3D ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WDGQIvbb9x +++ mktemp ++ local LAST_ERR=/tmp/tmp.G2OJafuY4C ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.WDGQIvbb9x ++ cat /tmp/tmp.G2OJafuY4C ++ rm /tmp/tmp.WDGQIvbb9x /tmp/tmp.G2OJafuY4C ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hhTvUac4Oo +++ mktemp ++ local LAST_ERR=/tmp/tmp.W3FZdgb6h0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hhTvUac4Oo ++ cat /tmp/tmp.W3FZdgb6h0 ++ rm /tmp/tmp.hhTvUac4Oo /tmp/tmp.W3FZdgb6h0 ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vQ5pSIPeJ2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.K9hsGnOmZA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vQ5pSIPeJ2 ++ cat /tmp/tmp.K9hsGnOmZA ++ rm /tmp/tmp.vQ5pSIPeJ2 /tmp/tmp.K9hsGnOmZA ++ return 0 + [[ initializing == \r\e\a\d\y ]] + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 20 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3AFdenirBQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.QZYjl2YeSI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.3AFdenirBQ ++ cat /tmp/tmp.QZYjl2YeSI ++ rm /tmp/tmp.3AFdenirBQ /tmp/tmp.QZYjl2YeSI ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bkF8v0CcqA +++ mktemp ++ local LAST_ERR=/tmp/tmp.UxV4eZ3q90 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.bkF8v0CcqA ++ cat /tmp/tmp.UxV4eZ3q90 ++ rm /tmp/tmp.bkF8v0CcqA /tmp/tmp.UxV4eZ3q90 ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.QzOB4HFSRR ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.mgI17QEtl5 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.QzOB4HFSRR +++++ cat /tmp/tmp.mgI17QEtl5 +++++ rm /tmp/tmp.QzOB4HFSRR /tmp/tmp.mgI17QEtl5 +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-haproxy ++++ return +++ local cluster_proxy=cross-site-source-haproxy +++ echo haproxy ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GQzjUVEW6X +++ mktemp ++ local LAST_ERR=/tmp/tmp.QI5l3CGYTt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.GQzjUVEW6X ++ cat /tmp/tmp.QI5l3CGYTt ++ rm /tmp/tmp.GQzjUVEW6X /tmp/tmp.QI5l3CGYTt ++ return 0 + [[ 2 == \2 ]] + run_mysql 'DELETE FROM myApp.myApp WHERE id=100500' '-h 10.7.138.253 -uroot -proot_password' + local 'command=DELETE FROM myApp.myApp WHERE id=100500' + local 'uri=-h 10.7.138.253 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.D2D0e6EJ87 +++ mktemp ++ local LAST_ERR=/tmp/tmp.PYgCLNBoox ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.D2D0e6EJ87 ++ cat /tmp/tmp.PYgCLNBoox ++ rm /tmp/tmp.D2D0e6EJ87 /tmp/tmp.PYgCLNBoox ++ return 0 + client_pod=pxc-client-599db5b9bc-mm5w6 + wait_pod pxc-client-599db5b9bc-mm5w6 + local pod=pxc-client-599db5b9bc-mm5w6 + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-mm5w6 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-mm5w6 condition met pxc-client-599db5b9bc-mm5w6.Ok + set +o xtrace + run_mysql 'DELETE FROM myApp.testSourceReplica WHERE id=100700' '-h 10.7.138.253 -uroot -proot_password' + local 'command=DELETE FROM myApp.testSourceReplica WHERE id=100700' + local 'uri=-h 10.7.138.253 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sfW1sQW85X +++ mktemp ++ local LAST_ERR=/tmp/tmp.DGLqBUIofq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sfW1sQW85X ++ cat /tmp/tmp.DGLqBUIofq ++ rm /tmp/tmp.sfW1sQW85X /tmp/tmp.DGLqBUIofq ++ return 0 + client_pod=pxc-client-599db5b9bc-mm5w6 + wait_pod pxc-client-599db5b9bc-mm5w6 + local pod=pxc-client-599db5b9bc-mm5w6 + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-mm5w6 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-mm5w6 condition met pxc-client-599db5b9bc-mm5w6.Ok + set +o xtrace + desc 'configure old replica as source' + set +o xtrace ----------------------------------------------------------------------------------- configure old replica as source ----------------------------------------------------------------------------------- + kubectl patch pxc cross-site-replica -n cross-site-replica-27495 --type=json '-p=[{'\''op'\'': '\''remove'\'', '\''path'\'': '\''/spec/pxc/replicationChannels/0'\''}]' perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + kubectl_bin patch pxc cross-site-replica -n cross-site-replica-27495 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": true}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.W5ybZ3mT7A ++ mktemp + local LAST_ERR=/tmp/tmp.dQikkvVZRB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-replica -n cross-site-replica-27495 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": true}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.W5ybZ3mT7A perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + cat /tmp/tmp.dQikkvVZRB + rm /tmp/tmp.W5ybZ3mT7A /tmp/tmp.dQikkvVZRB + return 0 + desc 'configure old source as replica' + set +o xtrace ----------------------------------------------------------------------------------- configure old source as replica ----------------------------------------------------------------------------------- + kubectl patch pxc cross-site-source -n cross-site-9668 --type=json '-p=[{'\''op'\'': '\''remove'\'', '\''path'\'': '\''/spec/pxc/replicationChannels/0'\''}]' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + kubectl_bin patch pxc cross-site-source -n cross-site-9668 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "10.7.129.40", "port": 3306, "weight": 100},{"host": "10.7.140.7", "port": 3306, "weight": 100},{"host": "10.7.143.112", "port": 3306, "weight": 100}]}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.Is6p5AiC57 ++ mktemp + local LAST_ERR=/tmp/tmp.nHQfneW6Ur + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc cross-site-source -n cross-site-9668 --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "replica_to_source", "isSource": false, "configuration": {"ssl": true, "sslSkipVerify": true, "ca": "/etc/mysql/ssl-internal/ca.crt"}, "sourcesList": [{"host": "10.7.129.40", "port": 3306, "weight": 100},{"host": "10.7.140.7", "port": 3306, "weight": 100},{"host": "10.7.143.112", "port": 3306, "weight": 100}]}]}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Is6p5AiC57 perconaxtradbcluster.pxc.percona.com/cross-site-source patched + cat /tmp/tmp.nHQfneW6Ur + rm /tmp/tmp.Is6p5AiC57 /tmp/tmp.nHQfneW6Ur + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.1LJGfvSmBV +++ mktemp ++ local LAST_ERR=/tmp/tmp.jNHK5qE9B8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1LJGfvSmBV ++ cat /tmp/tmp.jNHK5qE9B8 ++ rm /tmp/tmp.1LJGfvSmBV /tmp/tmp.jNHK5qE9B8 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3 --namespace=cross-site-replica-27495 ++ mktemp + local LAST_OUT=/tmp/tmp.Spi2vuhhGD ++ mktemp + local LAST_ERR=/tmp/tmp.N8MrHrKb7y + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3 --namespace=cross-site-replica-27495 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Spi2vuhhGD Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3" modified. + cat /tmp/tmp.N8MrHrKb7y + rm /tmp/tmp.Spi2vuhhGD /tmp/tmp.N8MrHrKb7y + return 0 + desc 'Write data to replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- Write data to replica cluster ----------------------------------------------------------------------------------- + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testReplicaSource (id int PRIMARY KEY);' '-h 10.7.129.40 -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testReplicaSource (id int PRIMARY KEY);' + local 'uri=-h 10.7.129.40 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1scnQmIzrg +++ mktemp ++ local LAST_ERR=/tmp/tmp.uEnfCoCjcO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1scnQmIzrg ++ cat /tmp/tmp.uEnfCoCjcO ++ rm /tmp/tmp.1scnQmIzrg /tmp/tmp.uEnfCoCjcO ++ return 0 + client_pod=pxc-client-599db5b9bc-5wznj + wait_pod pxc-client-599db5b9bc-5wznj + local pod=pxc-client-599db5b9bc-5wznj + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-5wznj ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-5wznj condition met pxc-client-599db5b9bc-5wznj.Ok + set +o xtrace + run_mysql 'INSERT myApp.testReplicaSource (id) VALUES (100800)' '-h 10.7.129.40 -uroot -proot_password' + local 'command=INSERT myApp.testReplicaSource (id) VALUES (100800)' + local 'uri=-h 10.7.129.40 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.froVfVCIaU +++ mktemp ++ local LAST_ERR=/tmp/tmp.EtBcJrg84D ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.froVfVCIaU ++ cat /tmp/tmp.EtBcJrg84D ++ rm /tmp/tmp.froVfVCIaU /tmp/tmp.EtBcJrg84D ++ return 0 + client_pod=pxc-client-599db5b9bc-5wznj + wait_pod pxc-client-599db5b9bc-5wznj + local pod=pxc-client-599db5b9bc-5wznj + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-5wznj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-5wznj condition met pxc-client-599db5b9bc-5wznj.Ok + set +o xtrace + sleep 15 + desc 'Check replication works between replica -> source' + set +o xtrace ----------------------------------------------------------------------------------- Check replication works between replica -> source ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.6KDaqYUr8O +++ mktemp ++ local LAST_ERR=/tmp/tmp.ax8EICN91Z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6KDaqYUr8O ++ cat /tmp/tmp.ax8EICN91Z ++ rm /tmp/tmp.6KDaqYUr8O /tmp/tmp.ax8EICN91Z ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3 --namespace=cross-site-9668 ++ mktemp + local LAST_OUT=/tmp/tmp.eNbvMsmael ++ mktemp + local LAST_ERR=/tmp/tmp.3YHPb4S4uh + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3 --namespace=cross-site-9668 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.eNbvMsmael Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1522-1ed4605b-21-cluster3" modified. + cat /tmp/tmp.3YHPb4S4uh + rm /tmp/tmp.eNbvMsmael /tmp/tmp.3YHPb4S4uh + return 0 + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 10.7.138.253 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.7.138.253 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-3-80.sql ']' + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 10.7.138.253 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.7.138.253 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.II0Vk9d7a1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.3Tlzt4ma2j ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.II0Vk9d7a1 ++ cat /tmp/tmp.3Tlzt4ma2j ++ rm /tmp/tmp.II0Vk9d7a1 /tmp/tmp.3Tlzt4ma2j ++ return 0 + client_pod=pxc-client-599db5b9bc-mm5w6 + wait_pod pxc-client-599db5b9bc-mm5w6 + local pod=pxc-client-599db5b9bc-mm5w6 + local max_retry=480 + local ns= ++ echo pxc-client-599db5b9bc-mm5w6 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-599db5b9bc-mm5w6 condition met pxc-client-599db5b9bc-mm5w6.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.nxIQyhbhUS/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-3.sql /tmp/tmp.nxIQyhbhUS/select-3.sql --- /mnt/jenkins/workspace/cloud-pxc-operator_PR-1522/e2e-tests/cross-site/compare/select-3.sql 2023-12-29 10:10:17.883164666 +0000 +++ /tmp/tmp.nxIQyhbhUS/select-3.sql 2023-12-29 11:37:49.424239013 +0000 @@ -1 +1,2 @@ -100800 +ERROR 1146 (42S02) at line 1: Table 'myApp.testReplicaSource' doesn't exist +command terminated with exit code 1