++ echo 'Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/logs/cross-site.log' Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/logs/cross-site.log ++ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP= ++ oc get projects error: No Auth Provider found for name "gcp" +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep '\-eks\-' ++ '[' ']' ++ EKS=0 +++ kubectl version -o json +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/bin/sed -r 's/[^0-9.]+//g' ++ KUBE_VERSION=1.20 +++ helm version -c +++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/' ++ HELM_VERSION=v3.8.1 ++ '[' v3 == v2 ']' + main + local source_cluster=cross-site-source + local replica_cluster=cross-site-replica + unset OPERATOR_NS + desc 'Create source cluster' + set +o xtrace ----------------------------------------------------------------------------------- Create source cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-10017 + local namespace=cross-site-10017 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + kubectl_bin get ns + '[' '!' -z '' ']' + kubectl_bin delete namespace cross-site-10017 + xargs kubectl delete ns + awk '{print$1}' + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^NAME' ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.SJOc1NypAz + local LAST_OUT=/tmp/tmp.bQWBZUfpLN ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.5xpkCKloAe + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.EPDOL2h3lC + local exit_status=0 + for i in '$(seq 0 2)' + kubectl delete namespace cross-site-10017 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl get ns + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace cross-site-10017 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.bQWBZUfpLN + cat /tmp/tmp.EPDOL2h3lC + rm /tmp/tmp.bQWBZUfpLN /tmp/tmp.EPDOL2h3lC + return 0 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace cross-site-10017 namespace "cross-site-5864" deleted namespace "cross-site-replica-1754" deleted + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + cat /tmp/tmp.SJOc1NypAz + cat /tmp/tmp.5xpkCKloAe Error from server (NotFound): namespaces "cross-site-10017" not found + rm /tmp/tmp.SJOc1NypAz /tmp/tmp.5xpkCKloAe + return 1 + : + wait_for_delete namespace/cross-site-10017 + local res=namespace/cross-site-10017 + set +o xtrace namespace/cross-site-10017 - Error from server (NotFound): namespaces "cross-site-10017" not found + kubectl_bin create namespace cross-site-10017 ++ mktemp + local LAST_OUT=/tmp/tmp.NsRN7ciRK5 ++ mktemp + local LAST_ERR=/tmp/tmp.EccLzchoMI + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create namespace cross-site-10017 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.NsRN7ciRK5 namespace/cross-site-10017 created + cat /tmp/tmp.EccLzchoMI + rm /tmp/tmp.NsRN7ciRK5 /tmp/tmp.EccLzchoMI + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.2kbdN1HPpy +++ mktemp ++ local LAST_ERR=/tmp/tmp.t3KFLYLzv9 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.2kbdN1HPpy ++ cat /tmp/tmp.t3KFLYLzv9 ++ rm /tmp/tmp.2kbdN1HPpy /tmp/tmp.t3KFLYLzv9 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site --namespace=cross-site-10017 ++ mktemp + local LAST_OUT=/tmp/tmp.edgUcfTd6I ++ mktemp + local LAST_ERR=/tmp/tmp.9oOIe7RVaP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site --namespace=cross-site-10017 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.edgUcfTd6I Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site" modified. + cat /tmp/tmp.9oOIe7RVaP + rm /tmp/tmp.edgUcfTd6I /tmp/tmp.9oOIe7RVaP + return 0 + deploy_operator + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.v7vWBtHHYU ++ mktemp + local LAST_ERR=/tmp/tmp.OzVURx8gfr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/crd.yaml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.v7vWBtHHYU customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbbackups.pxc.percona.com configured + cat /tmp/tmp.OzVURx8gfr + rm /tmp/tmp.v7vWBtHHYU /tmp/tmp.OzVURx8gfr + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=pxc-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.E8N7PfsBpQ ++ mktemp + local LAST_ERR=/tmp/tmp.AQgpeaHSqP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.E8N7PfsBpQ role.rbac.authorization.k8s.io/percona-xtradb-cluster-operator created serviceaccount/percona-xtradb-cluster-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator created + cat /tmp/tmp.AQgpeaHSqP + rm /tmp/tmp.E8N7PfsBpQ /tmp/tmp.AQgpeaHSqP + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/operator.yaml + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - ++ mktemp + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a^' + local LAST_OUT=/tmp/tmp.5NSFQSRMRn ++ mktemp + local LAST_ERR=/tmp/tmp.h38F7f8vKU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.5NSFQSRMRn deployment.apps/percona-xtradb-cluster-operator created + cat /tmp/tmp.h38F7f8vKU + rm /tmp/tmp.5NSFQSRMRn /tmp/tmp.h38F7f8vKU + return 0 + sleep 10 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5U4pJXvRn2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gVmblXZDcH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.5U4pJXvRn2 ++ cat /tmp/tmp.gVmblXZDcH ++ rm /tmp/tmp.5U4pJXvRn2 /tmp/tmp.gVmblXZDcH ++ return 0 + wait_pod percona-xtradb-cluster-operator-64649ffc9c-tsbls 480 '' + local pod=percona-xtradb-cluster-operator-64649ffc9c-tsbls + local max_retry=480 + local ns= ++ echo percona-xtradb-cluster-operator-64649ffc9c-tsbls ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace percona-xtradb-cluster-operator-64649ffc9c-tsbls.Ok + sleep 3 + apply_secrets + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.fYlh5wMeah ++ mktemp + local LAST_ERR=/tmp/tmp.l3WryX8fSP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.fYlh5wMeah secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.l3WryX8fSP + rm /tmp/tmp.fYlh5wMeah /tmp/tmp.l3WryX8fSP + return 0 + spinup_pxc cross-site-source /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/conf/cross-site-source.yml + local cluster=cross-site-source + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/conf/cross-site-source.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.DoiRNJCq9V ++ mktemp + local LAST_ERR=/tmp/tmp.PjE1Jkiw5V + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.DoiRNJCq9V secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.PjE1Jkiw5V + rm /tmp/tmp.DoiRNJCq9V /tmp/tmp.PjE1Jkiw5V + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-10017~ + local LAST_OUT=/tmp/tmp.TK9VKy9Xdd + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1-11-0#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + local LAST_ERR=/tmp/tmp.PrOqqL3z1R + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.TK9VKy9Xdd deployment.apps/pxc-client created + cat /tmp/tmp.PrOqqL3z1R + rm /tmp/tmp.TK9VKy9Xdd /tmp/tmp.PrOqqL3z1R + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/conf/cross-site-source.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/conf/cross-site-source.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Pwow1P01Cq + /usr/bin/sed -e 's#apply:.*#apply: Never#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/conf/cross-site-source.yml + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-10017~ + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1-11-0#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_ERR=/tmp/tmp.DagQRT8Clv + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local exit_status=0 + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + for i in '$(seq 0 2)' + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.Pwow1P01Cq perconaxtradbcluster.pxc.percona.com/cross-site-source created + cat /tmp/tmp.DagQRT8Clv + rm /tmp/tmp.Pwow1P01Cq /tmp/tmp.DagQRT8Clv + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy cross-site-source ++ local target_cluster=cross-site-source +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rnLlG3Zc4L ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BAYPJsZcTU +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.rnLlG3Zc4L +++ cat /tmp/tmp.BAYPJsZcTU +++ rm /tmp/tmp.rnLlG3Zc4L /tmp/tmp.BAYPJsZcTU +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.aIY8kcfusm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ePUHifx8f4 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.aIY8kcfusm +++ cat /tmp/tmp.ePUHifx8f4 +++ rm /tmp/tmp.aIY8kcfusm /tmp/tmp.ePUHifx8f4 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo cross-site-source-proxysql ++ return + local proxy=cross-site-source-proxysql + wait_for_running cross-site-source-proxysql 1 + local name=cross-site-source-proxysql + let last_pod=0 + : + local max_retry=480 ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-proxysql-0 480 + local pod=cross-site-source-proxysql-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo cross-site-source-proxysql-0 + local container=proxysql + set +o xtrace cross-site-source-proxysql-0.........Ok + wait_for_running cross-site-source-pxc 3 + local name=cross-site-source-pxc + let last_pod=2 + local max_retry=480 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-0 480 + local pod=cross-site-source-pxc-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo cross-site-source-pxc-0 + local container=pxc + set +o xtrace cross-site-source-pxc-0.......................Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-1 480 + local pod=cross-site-source-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo cross-site-source-pxc-1 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace cross-site-source-pxc-1...............................Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-source-pxc-2 480 + local pod=cross-site-source-pxc-2 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo cross-site-source-pxc-2 + local container=pxc + set +o xtrace cross-site-source-pxc-2...........................Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h cross-site-source-proxysql -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h cross-site-source-proxysql -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rkxdmVk3F5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.2LGxpSPxvn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.rkxdmVk3F5 ++ cat /tmp/tmp.2LGxpSPxvn ++ rm /tmp/tmp.rkxdmVk3F5 /tmp/tmp.2LGxpSPxvn ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-vjk86 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h cross-site-source-proxysql -uroot -proot_password' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h cross-site-source-proxysql -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mYrOJCkIAB +++ mktemp ++ local LAST_ERR=/tmp/tmp.NxmPf18G3P ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.mYrOJCkIAB ++ cat /tmp/tmp.NxmPf18G3P ++ rm /tmp/tmp.mYrOJCkIAB /tmp/tmp.NxmPf18G3P ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-vjk86 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IzpbQcjxme +++ mktemp ++ local LAST_ERR=/tmp/tmp.hKvveQ8l8J ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.IzpbQcjxme ++ cat /tmp/tmp.hKvveQ8l8J ++ rm /tmp/tmp.IzpbQcjxme /tmp/tmp.hKvveQ8l8J ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-vjk86 + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.K32pxtTtiP/select-1.sql ']' + sleep 20 + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-0.cross-site-source-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SJ38CQHHHc +++ mktemp ++ local LAST_ERR=/tmp/tmp.soR9aIEehl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.SJ38CQHHHc ++ cat /tmp/tmp.soR9aIEehl ++ rm /tmp/tmp.SJ38CQHHHc /tmp/tmp.soR9aIEehl ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-vjk86 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.K32pxtTtiP/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-1.cross-site-source-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AxdavCOHPz +++ mktemp ++ local LAST_ERR=/tmp/tmp.W3XGQRxlZw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.AxdavCOHPz ++ cat /tmp/tmp.W3XGQRxlZw ++ rm /tmp/tmp.AxdavCOHPz /tmp/tmp.W3XGQRxlZw ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-vjk86 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.K32pxtTtiP/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.K32pxtTtiP/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-source-pxc-2.cross-site-source-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hkQLCz8rFU +++ mktemp ++ local LAST_ERR=/tmp/tmp.4mBoS5wtcU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.hkQLCz8rFU ++ cat /tmp/tmp.4mBoS5wtcU ++ rm /tmp/tmp.hkQLCz8rFU /tmp/tmp.4mBoS5wtcU ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-vjk86 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.K32pxtTtiP/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.K32pxtTtiP/select-1.sql ++ is_keyring_plugin_in_use cross-site-source ++ local cluster=cross-site-source ++ kubectl_bin exec -it cross-site-source-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WvHng9Jogg +++ mktemp ++ local LAST_ERR=/tmp/tmp.5Qllcae9XG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl exec -it cross-site-source-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.WvHng9Jogg ++ cat /tmp/tmp.5Qllcae9XG Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.WvHng9Jogg /tmp/tmp.5Qllcae9XG ++ return 0 + '[' '' ']' + sleep 60 ++ get_service_ip cross-site-source-pxc-0 ++ local service=cross-site-source-pxc-0 ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dYmkfQQaLV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nMARSsKYLj +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.dYmkfQQaLV +++ cat /tmp/tmp.nMARSsKYLj +++ rm /tmp/tmp.dYmkfQQaLV /tmp/tmp.nMARSsKYLj +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bhY01dU2fa +++ mktemp ++ local LAST_ERR=/tmp/tmp.FDAVDiwEgr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.bhY01dU2fa ++ cat /tmp/tmp.FDAVDiwEgr ++ rm /tmp/tmp.bhY01dU2fa /tmp/tmp.FDAVDiwEgr ++ return 0 ++ return + source_endpoint0=10.131.251.87 ++ get_service_ip cross-site-source-pxc-1 ++ local service=cross-site-source-pxc-1 ++ grep -q NotFound ++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9A1xjJm2uE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.drcW0bQpPs +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.9A1xjJm2uE +++ cat /tmp/tmp.drcW0bQpPs +++ rm /tmp/tmp.9A1xjJm2uE /tmp/tmp.drcW0bQpPs +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V9tAW5DYut +++ mktemp ++ local LAST_ERR=/tmp/tmp.aGxdczX8dq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get service/cross-site-source-pxc-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.V9tAW5DYut ++ cat /tmp/tmp.aGxdczX8dq ++ rm /tmp/tmp.V9tAW5DYut /tmp/tmp.aGxdczX8dq ++ return 0 ++ return + source_endpoint1=10.131.243.40 ++ get_service_ip cross-site-source-pxc-2 ++ local service=cross-site-source-pxc-2 ++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oFbhTd9jMP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DlRv2yr6kC +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.oFbhTd9jMP +++ cat /tmp/tmp.DlRv2yr6kC +++ rm /tmp/tmp.oFbhTd9jMP /tmp/tmp.DlRv2yr6kC +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5j42RzIqR5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZwFAD0zFCv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get service/cross-site-source-pxc-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.5j42RzIqR5 ++ cat /tmp/tmp.ZwFAD0zFCv ++ rm /tmp/tmp.5j42RzIqR5 /tmp/tmp.ZwFAD0zFCv ++ return 0 ++ return + source_endpoint2=10.131.240.192 ++ get_proxy_primary '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-source-proxysql-0 ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=cross-site-source-proxysql-0 +++ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status="ONLINE";' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-source-proxysql-0 proxysql +++ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status="ONLINE";' +++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' +++ local pod=cross-site-source-proxysql-0 +++ local container_name=proxysql +++ set +o xtrace ++ local ip=cross-site-source-pxc-0.cross-site-source-pxc.cross-site-10017.svc.cluster.local +++ wc -l +++ echo cross-site-source-pxc-0.cross-site-source-pxc.cross-site-10017.svc.cluster.local ++ '[' 1 '!=' 1 ']' ++ cut -d. -f1 ++ echo cross-site-source-pxc-0.cross-site-source-pxc.cross-site-10017.svc.cluster.local + source_primary=cross-site-source-pxc-0 ++ get_service_ip cross-site-source-pxc-0 ++ local service=cross-site-source-pxc-0 ++ grep -q NotFound ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PBVmkBhFm0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3Qx0ndYEJg +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.PBVmkBhFm0 +++ cat /tmp/tmp.3Qx0ndYEJg +++ rm /tmp/tmp.PBVmkBhFm0 /tmp/tmp.3Qx0ndYEJg +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7VyrvHKYXT +++ mktemp ++ local LAST_ERR=/tmp/tmp.GIVd4r3FyR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get service/cross-site-source-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.7VyrvHKYXT ++ cat /tmp/tmp.GIVd4r3FyR ++ rm /tmp/tmp.7VyrvHKYXT /tmp/tmp.GIVd4r3FyR ++ return 0 ++ return + source_primary_endpoint=10.131.251.87 + kubectl_bin patch pxc cross-site-source --type=merge --patch '{"spec": {"pxc":{"replicationChannels": [{"name":"source_to_replica", "isSource": true}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.lk6cVfFmLn ++ mktemp + local LAST_ERR=/tmp/tmp.ZeueNLKw2k + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch pxc cross-site-source --type=merge --patch '{"spec": {"pxc":{"replicationChannels": [{"name":"source_to_replica", "isSource": true}]}}}' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.lk6cVfFmLn perconaxtradbcluster.pxc.percona.com/cross-site-source patched + cat /tmp/tmp.ZeueNLKw2k + rm /tmp/tmp.lk6cVfFmLn /tmp/tmp.ZeueNLKw2k + return 0 ++ echo -n new_password ++ base64 + kubectl_bin patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.uhom0COfyz ++ mktemp + local LAST_ERR=/tmp/tmp.EySJoHeP2R + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.uhom0COfyz secret/my-cluster-secrets patched + cat /tmp/tmp.EySJoHeP2R + rm /tmp/tmp.uhom0COfyz /tmp/tmp.EySJoHeP2R + return 0 + sleep 15 + wait_cluster_consistency cross-site-source 3 2 + local cluster_name=cross-site-source + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + sleep 7 ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AyE4v2Vqlz +++ mktemp ++ local LAST_ERR=/tmp/tmp.PgYhPYtCRQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.AyE4v2Vqlz ++ cat /tmp/tmp.PgYhPYtCRQ ++ rm /tmp/tmp.AyE4v2Vqlz /tmp/tmp.PgYhPYtCRQ ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QyWYobbz61 +++ mktemp ++ local LAST_ERR=/tmp/tmp.io4vbXtCxv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.QyWYobbz61 ++ cat /tmp/tmp.io4vbXtCxv ++ rm /tmp/tmp.QyWYobbz61 /tmp/tmp.io4vbXtCxv ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-source +++ local cluster_name=cross-site-source ++++ get_proxy cross-site-source ++++ local target_cluster=cross-site-source +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.LETwuM8eU5 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.qQgqvI7fNv +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ cat /tmp/tmp.LETwuM8eU5 +++++ cat /tmp/tmp.qQgqvI7fNv +++++ rm /tmp/tmp.LETwuM8eU5 /tmp/tmp.qQgqvI7fNv +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.2eNEjDWgn2 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.m0CcNNG9ir +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ kubectl get pxc cross-site-source -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ cat /tmp/tmp.2eNEjDWgn2 +++++ cat /tmp/tmp.m0CcNNG9ir +++++ rm /tmp/tmp.2eNEjDWgn2 /tmp/tmp.m0CcNNG9ir +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-source-proxysql ++++ return +++ local cluster_proxy=cross-site-source-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5HloUOyFsk +++ mktemp ++ local LAST_ERR=/tmp/tmp.P9JzV08XHD ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc cross-site-source -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.5HloUOyFsk ++ cat /tmp/tmp.P9JzV08XHD ++ rm /tmp/tmp.5HloUOyFsk /tmp/tmp.P9JzV08XHD ++ return 0 + [[ 2 == \2 ]] + desc 'Write data to source cluster' + set +o xtrace ----------------------------------------------------------------------------------- Write data to source cluster ----------------------------------------------------------------------------------- + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testSourceReplica (id int PRIMARY KEY);' '-h 10.131.251.87 -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testSourceReplica (id int PRIMARY KEY);' + local 'uri=-h 10.131.251.87 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pZCI223Qoe +++ mktemp ++ local LAST_ERR=/tmp/tmp.yL9i4u5C3O ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.pZCI223Qoe ++ cat /tmp/tmp.yL9i4u5C3O ++ rm /tmp/tmp.pZCI223Qoe /tmp/tmp.yL9i4u5C3O ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-vjk86 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + run_mysql 'INSERT myApp.testSourceReplica (id) VALUES (100700)' '-h 10.131.251.87 -uroot -proot_password' + local 'command=INSERT myApp.testSourceReplica (id) VALUES (100700)' + local 'uri=-h 10.131.251.87 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Vlrj7VRQX1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Trovpr5X6n ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Vlrj7VRQX1 ++ cat /tmp/tmp.Trovpr5X6n ++ rm /tmp/tmp.Vlrj7VRQX1 /tmp/tmp.Trovpr5X6n ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-vjk86 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + desc 'Create replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- Create replica cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-replica-27981 0 + local namespace=cross-site-replica-27981 + local skip_clean_namespace=0 + [[ 1 == 1 ]] + [[ -z 0 ]] + '[' '!' -z '' ']' + kubectl_bin delete namespace cross-site-replica-27981 ++ mktemp + local LAST_OUT=/tmp/tmp.JNPBu8bBAw ++ mktemp + local LAST_ERR=/tmp/tmp.xDXbSKTuJE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete namespace cross-site-replica-27981 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace cross-site-replica-27981 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace cross-site-replica-27981 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + cat /tmp/tmp.JNPBu8bBAw + cat /tmp/tmp.xDXbSKTuJE Error from server (NotFound): namespaces "cross-site-replica-27981" not found + rm /tmp/tmp.JNPBu8bBAw /tmp/tmp.xDXbSKTuJE + return 1 + : + wait_for_delete namespace/cross-site-replica-27981 + local res=namespace/cross-site-replica-27981 + set +o xtrace namespace/cross-site-replica-27981 - Error from server (NotFound): namespaces "cross-site-replica-27981" not found + kubectl_bin create namespace cross-site-replica-27981 ++ mktemp + local LAST_OUT=/tmp/tmp.q3LCTUpjiX ++ mktemp + local LAST_ERR=/tmp/tmp.rU3QjjTiXl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create namespace cross-site-replica-27981 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.q3LCTUpjiX namespace/cross-site-replica-27981 created + cat /tmp/tmp.rU3QjjTiXl + rm /tmp/tmp.q3LCTUpjiX /tmp/tmp.rU3QjjTiXl + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.wSquRfeaUE +++ mktemp ++ local LAST_ERR=/tmp/tmp.MOcPgS4cp6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.wSquRfeaUE ++ cat /tmp/tmp.MOcPgS4cp6 ++ rm /tmp/tmp.wSquRfeaUE /tmp/tmp.MOcPgS4cp6 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site --namespace=cross-site-replica-27981 ++ mktemp + local LAST_OUT=/tmp/tmp.sIrs7Hviyl ++ mktemp + local LAST_ERR=/tmp/tmp.7qNIK2JZoS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site --namespace=cross-site-replica-27981 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.sIrs7Hviyl Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site" modified. + cat /tmp/tmp.7qNIK2JZoS + rm /tmp/tmp.sIrs7Hviyl /tmp/tmp.7qNIK2JZoS + return 0 + deploy_operator + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.mBhKGQaW3A ++ mktemp + local LAST_ERR=/tmp/tmp.W8mqC66UR8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/crd.yaml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.mBhKGQaW3A customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbbackups.pxc.percona.com configured + cat /tmp/tmp.W8mqC66UR8 + rm /tmp/tmp.mBhKGQaW3A /tmp/tmp.W8mqC66UR8 + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=pxc-operator + local rbac=rbac + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.7nNupfl2KI ++ mktemp + local LAST_ERR=/tmp/tmp.zAzhJBkaV5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.7nNupfl2KI role.rbac.authorization.k8s.io/percona-xtradb-cluster-operator created serviceaccount/percona-xtradb-cluster-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator created + cat /tmp/tmp.zAzhJBkaV5 + rm /tmp/tmp.7nNupfl2KI /tmp/tmp.zAzhJBkaV5 + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/operator.yaml + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a^' ++ mktemp + local LAST_OUT=/tmp/tmp.PMNq56RrxF ++ mktemp + local LAST_ERR=/tmp/tmp.pA0IRmTcrq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.PMNq56RrxF deployment.apps/percona-xtradb-cluster-operator created + cat /tmp/tmp.pA0IRmTcrq + rm /tmp/tmp.PMNq56RrxF /tmp/tmp.pA0IRmTcrq + return 0 + sleep 10 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0uUWqL0in9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.O1tZxRserE ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.0uUWqL0in9 ++ cat /tmp/tmp.O1tZxRserE ++ rm /tmp/tmp.0uUWqL0in9 /tmp/tmp.O1tZxRserE ++ return 0 + wait_pod percona-xtradb-cluster-operator-64649ffc9c-n2tmv 480 '' + local pod=percona-xtradb-cluster-operator-64649ffc9c-n2tmv + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo percona-xtradb-cluster-operator-64649ffc9c-n2tmv + local container= + set +o xtrace percona-xtradb-cluster-operator-64649ffc9c-n2tmv.Ok + sleep 3 + apply_secrets + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xaaNQElWuW ++ mktemp + local LAST_ERR=/tmp/tmp.PydQDtWBhx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.xaaNQElWuW secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.PydQDtWBhx + rm /tmp/tmp.xaaNQElWuW /tmp/tmp.PydQDtWBhx + return 0 + spinup_pxc cross-site-replica /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/conf/cross-site-replica.yml + local cluster=cross-site-replica + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/conf/cross-site-replica.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.KV7fKPgVw1 ++ mktemp + local LAST_ERR=/tmp/tmp.ydWIJehDYf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.KV7fKPgVw1 secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.ydWIJehDYf + rm /tmp/tmp.KV7fKPgVw1 /tmp/tmp.ydWIJehDYf + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + kubectl_bin apply -f - ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1-11-0#' + local LAST_OUT=/tmp/tmp.NvHx9hKpi9 ++ mktemp + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + local LAST_ERR=/tmp/tmp.eBxv3fDN95 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-10017~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.NvHx9hKpi9 deployment.apps/pxc-client created + cat /tmp/tmp.eBxv3fDN95 + rm /tmp/tmp.NvHx9hKpi9 /tmp/tmp.eBxv3fDN95 + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/conf/cross-site-replica.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/conf/cross-site-replica.yml ++ mktemp + local LAST_OUT=/tmp/tmp.G6gQ7sJ7q4 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1-11-0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.cross-site-10017~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/conf/cross-site-replica.yml + local LAST_ERR=/tmp/tmp.HbCHcKuP52 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.G6gQ7sJ7q4 perconaxtradbcluster.pxc.percona.com/cross-site-replica created + cat /tmp/tmp.HbCHcKuP52 + rm /tmp/tmp.G6gQ7sJ7q4 /tmp/tmp.HbCHcKuP52 + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy cross-site-replica ++ local target_cluster=cross-site-replica +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ZmvJlxeewb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sI9fcL1hkE +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.ZmvJlxeewb +++ cat /tmp/tmp.sI9fcL1hkE +++ rm /tmp/tmp.ZmvJlxeewb /tmp/tmp.sI9fcL1hkE +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uHPDh6qW8w ++++ mktemp +++ local LAST_ERR=/tmp/tmp.n3YRxl1TJO +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.uHPDh6qW8w +++ cat /tmp/tmp.n3YRxl1TJO +++ rm /tmp/tmp.uHPDh6qW8w /tmp/tmp.n3YRxl1TJO +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo cross-site-replica-proxysql ++ return + local proxy=cross-site-replica-proxysql + wait_for_running cross-site-replica-proxysql 1 + local name=cross-site-replica-proxysql + let last_pod=0 + : + local max_retry=480 ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-proxysql-0 480 + local pod=cross-site-replica-proxysql-0 + local max_retry=480 + local ns= ++ echo cross-site-replica-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace cross-site-replica-proxysql-0.............Ok + wait_for_running cross-site-replica-pxc 3 + local name=cross-site-replica-pxc + let last_pod=2 + local max_retry=480 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-0 480 + local pod=cross-site-replica-pxc-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo cross-site-replica-pxc-0 + local container=pxc + set +o xtrace cross-site-replica-pxc-0...................Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-1 480 + local pod=cross-site-replica-pxc-1 + local max_retry=480 + local ns= ++ echo cross-site-replica-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace cross-site-replica-pxc-1...................................Ok + for i in '$(seq 0 $last_pod)' + wait_pod cross-site-replica-pxc-2 480 + local pod=cross-site-replica-pxc-2 + local max_retry=480 + local ns= ++ echo cross-site-replica-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace cross-site-replica-pxc-2.....................................Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h cross-site-replica-proxysql -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h cross-site-replica-proxysql -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mR5mtDpDRp +++ mktemp ++ local LAST_ERR=/tmp/tmp.opwq2HRIjB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.mR5mtDpDRp ++ cat /tmp/tmp.opwq2HRIjB ++ rm /tmp/tmp.mR5mtDpDRp /tmp/tmp.opwq2HRIjB ++ return 0 + client_pod=pxc-client-5d749ff8b6-5t9cj + wait_pod pxc-client-5d749ff8b6-5t9cj + local pod=pxc-client-5d749ff8b6-5t9cj + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-5t9cj + local container= + set +o xtrace pxc-client-5d749ff8b6-5t9cj.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h cross-site-replica-proxysql -uroot -proot_password' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h cross-site-replica-proxysql -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0PBe5ThQrs +++ mktemp ++ local LAST_ERR=/tmp/tmp.Sa1ljaMLL8 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.0PBe5ThQrs ++ cat /tmp/tmp.Sa1ljaMLL8 ++ rm /tmp/tmp.0PBe5ThQrs /tmp/tmp.Sa1ljaMLL8 ++ return 0 + client_pod=pxc-client-5d749ff8b6-5t9cj + wait_pod pxc-client-5d749ff8b6-5t9cj + local pod=pxc-client-5d749ff8b6-5t9cj + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-5t9cj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-5t9cj.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-0.cross-site-replica-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PHpski4H4y +++ mktemp ++ local LAST_ERR=/tmp/tmp.xh0nmzq8JO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.PHpski4H4y ++ cat /tmp/tmp.xh0nmzq8JO ++ rm /tmp/tmp.PHpski4H4y /tmp/tmp.xh0nmzq8JO ++ return 0 + client_pod=pxc-client-5d749ff8b6-5t9cj + wait_pod pxc-client-5d749ff8b6-5t9cj + local pod=pxc-client-5d749ff8b6-5t9cj + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-5t9cj ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-5d749ff8b6-5t9cj.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.K32pxtTtiP/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.K32pxtTtiP/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-1.cross-site-replica-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cMMxbTWVHW +++ mktemp ++ local LAST_ERR=/tmp/tmp.Yyi879cqGZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.cMMxbTWVHW ++ cat /tmp/tmp.Yyi879cqGZ ++ rm /tmp/tmp.cMMxbTWVHW /tmp/tmp.Yyi879cqGZ ++ return 0 + client_pod=pxc-client-5d749ff8b6-5t9cj + wait_pod pxc-client-5d749ff8b6-5t9cj + local pod=pxc-client-5d749ff8b6-5t9cj + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-5t9cj ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-5d749ff8b6-5t9cj.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.K32pxtTtiP/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.K32pxtTtiP/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h cross-site-replica-pxc-2.cross-site-replica-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.taE723bUe2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.2Sb7NhFjmx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.taE723bUe2 ++ cat /tmp/tmp.2Sb7NhFjmx ++ rm /tmp/tmp.taE723bUe2 /tmp/tmp.2Sb7NhFjmx ++ return 0 + client_pod=pxc-client-5d749ff8b6-5t9cj + wait_pod pxc-client-5d749ff8b6-5t9cj + local pod=pxc-client-5d749ff8b6-5t9cj + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-5t9cj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-5t9cj.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.K32pxtTtiP/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-1.sql /tmp/tmp.K32pxtTtiP/select-1.sql ++ is_keyring_plugin_in_use cross-site-replica ++ local cluster=cross-site-replica ++ kubectl_bin exec -it cross-site-replica-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9Ebo6yKuJa +++ mktemp ++ local LAST_ERR=/tmp/tmp.83pK6oVyyi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl exec -it cross-site-replica-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.9Ebo6yKuJa ++ cat /tmp/tmp.83pK6oVyyi Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.9Ebo6yKuJa /tmp/tmp.83pK6oVyyi ++ return 0 + '[' '' ']' + sleep 60 ++ get_service_ip cross-site-replica-pxc-0 ++ local service=cross-site-replica-pxc-0 ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LfvaB5eOLm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Uum4cMlrH3 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.LfvaB5eOLm +++ cat /tmp/tmp.Uum4cMlrH3 +++ rm /tmp/tmp.LfvaB5eOLm /tmp/tmp.Uum4cMlrH3 +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LrpVMjXWTQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.MrAJkv9vih ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.LrpVMjXWTQ ++ cat /tmp/tmp.MrAJkv9vih ++ rm /tmp/tmp.LrpVMjXWTQ /tmp/tmp.MrAJkv9vih ++ return 0 ++ return + replica_endpoint0=10.131.241.25 ++ get_service_ip cross-site-replica-pxc-1 ++ local service=cross-site-replica-pxc-1 ++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1ttZZ6Iufm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fhpdGr56Xn +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.1ttZZ6Iufm +++ cat /tmp/tmp.fhpdGr56Xn +++ rm /tmp/tmp.1ttZZ6Iufm /tmp/tmp.fhpdGr56Xn +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4sMP2VzELc +++ mktemp ++ local LAST_ERR=/tmp/tmp.l5imfSKgJj ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get service/cross-site-replica-pxc-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.4sMP2VzELc ++ cat /tmp/tmp.l5imfSKgJj ++ rm /tmp/tmp.4sMP2VzELc /tmp/tmp.l5imfSKgJj ++ return 0 ++ return + replica_endpoint1=10.131.252.27 ++ get_service_ip cross-site-replica-pxc-2 ++ local service=cross-site-replica-pxc-2 ++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9lcrTFVOEs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DNQL0E6Si9 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.9lcrTFVOEs +++ cat /tmp/tmp.DNQL0E6Si9 +++ rm /tmp/tmp.9lcrTFVOEs /tmp/tmp.DNQL0E6Si9 +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.y2gPA7aw8H +++ mktemp ++ local LAST_ERR=/tmp/tmp.AwdjtfBRTl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get service/cross-site-replica-pxc-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.y2gPA7aw8H ++ cat /tmp/tmp.AwdjtfBRTl ++ rm /tmp/tmp.y2gPA7aw8H /tmp/tmp.AwdjtfBRTl ++ return 0 ++ return + replica_endpoint2=10.131.245.142 ++ get_proxy_primary '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-replica-proxysql-0 ++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' ++ local pod=cross-site-replica-proxysql-0 +++ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status="ONLINE";' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' cross-site-replica-proxysql-0 proxysql +++ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status="ONLINE";' +++ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' +++ local pod=cross-site-replica-proxysql-0 +++ local container_name=proxysql +++ set +o xtrace ++ local ip=cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-replica-27981.svc.cluster.local +++ echo cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-replica-27981.svc.cluster.local +++ wc -l ++ '[' 1 '!=' 1 ']' ++ echo cross-site-replica-pxc-0.cross-site-replica-pxc.cross-site-replica-27981.svc.cluster.local ++ cut -d. -f1 + replica_primary=cross-site-replica-pxc-0 ++ get_service_ip cross-site-replica-pxc-0 ++ local service=cross-site-replica-pxc-0 ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yPlmJRz4xH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.eE77QOfLVb +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.yPlmJRz4xH +++ cat /tmp/tmp.eE77QOfLVb +++ rm /tmp/tmp.yPlmJRz4xH /tmp/tmp.eE77QOfLVb +++ return 0 ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Apx4zvEPKL +++ mktemp ++ local LAST_ERR=/tmp/tmp.GQVdMOO6Zd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get service/cross-site-replica-pxc-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Apx4zvEPKL ++ cat /tmp/tmp.GQVdMOO6Zd ++ rm /tmp/tmp.Apx4zvEPKL /tmp/tmp.GQVdMOO6Zd ++ return 0 ++ return + replica_primary_endpoint=10.131.241.25 + run_mysql 'DELETE FROM myApp.myApp WHERE id=100500' '-h 10.131.241.25 -uroot -proot_password' + local 'command=DELETE FROM myApp.myApp WHERE id=100500' + local 'uri=-h 10.131.241.25 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jPebUmYNQO +++ mktemp ++ local LAST_ERR=/tmp/tmp.eNDgj13xNM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.jPebUmYNQO ++ cat /tmp/tmp.eNDgj13xNM ++ rm /tmp/tmp.jPebUmYNQO /tmp/tmp.eNDgj13xNM ++ return 0 + client_pod=pxc-client-5d749ff8b6-5t9cj + wait_pod pxc-client-5d749ff8b6-5t9cj + local pod=pxc-client-5d749ff8b6-5t9cj + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-5t9cj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-5t9cj.Ok + set +o xtrace + kubectl_bin patch pxc cross-site-replica --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": false, "sourcesList": [{"host": "10.131.251.87", "port": 3306, "weight": 100},{"host": "10.131.243.40", "port": 3306, "weight": 100},{"host": "10.131.240.192", "port": 3306, "weight": 100}]}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.Jr9TjPgZRb ++ mktemp + local LAST_ERR=/tmp/tmp.JWTeZ9XGvR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch pxc cross-site-replica --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": false, "sourcesList": [{"host": "10.131.251.87", "port": 3306, "weight": 100},{"host": "10.131.243.40", "port": 3306, "weight": 100},{"host": "10.131.240.192", "port": 3306, "weight": 100}]}]}}}' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.Jr9TjPgZRb perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + cat /tmp/tmp.JWTeZ9XGvR + rm /tmp/tmp.Jr9TjPgZRb /tmp/tmp.JWTeZ9XGvR + return 0 ++ echo -n new_password ++ base64 + kubectl_bin patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' ++ mktemp + local LAST_OUT=/tmp/tmp.qPHfoSh1zK ++ mktemp + local LAST_ERR=/tmp/tmp.zPIDGhDVRl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch secret my-cluster-secrets -p '{"data":{"replication": "bmV3X3Bhc3N3b3Jk"}}' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.qPHfoSh1zK secret/my-cluster-secrets patched + cat /tmp/tmp.zPIDGhDVRl + rm /tmp/tmp.qPHfoSh1zK /tmp/tmp.zPIDGhDVRl + return 0 + sleep 15 + wait_cluster_consistency cross-site-replica 3 2 + local cluster_name=cross-site-replica + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + sleep 7 ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UglKuvxQMU +++ mktemp ++ local LAST_ERR=/tmp/tmp.qjVgbXaCXY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.UglKuvxQMU ++ cat /tmp/tmp.qjVgbXaCXY ++ rm /tmp/tmp.UglKuvxQMU /tmp/tmp.qjVgbXaCXY ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7dYN3AD6rF +++ mktemp ++ local LAST_ERR=/tmp/tmp.eePs8dyydW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.7dYN3AD6rF ++ cat /tmp/tmp.eePs8dyydW ++ rm /tmp/tmp.7dYN3AD6rF /tmp/tmp.eePs8dyydW ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine cross-site-replica +++ local cluster_name=cross-site-replica ++++ get_proxy cross-site-replica ++++ local target_cluster=cross-site-replica +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.CIj8jCeWO9 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.aWJi5XvYLa +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ cat /tmp/tmp.CIj8jCeWO9 +++++ cat /tmp/tmp.aWJi5XvYLa +++++ rm /tmp/tmp.CIj8jCeWO9 /tmp/tmp.aWJi5XvYLa +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.vMyjSRh89i ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.F166mCWXfR +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ kubectl get pxc cross-site-replica -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ cat /tmp/tmp.vMyjSRh89i +++++ cat /tmp/tmp.F166mCWXfR +++++ rm /tmp/tmp.vMyjSRh89i /tmp/tmp.F166mCWXfR +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo cross-site-replica-proxysql ++++ return +++ local cluster_proxy=cross-site-replica-proxysql +++ echo proxysql ++ kubectl_bin get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ezJPmldy4R +++ mktemp ++ local LAST_ERR=/tmp/tmp.qEHshNCRPN ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pxc cross-site-replica -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.ezJPmldy4R ++ cat /tmp/tmp.qEHshNCRPN ++ rm /tmp/tmp.ezJPmldy4R /tmp/tmp.qEHshNCRPN ++ return 0 + [[ 2 == \2 ]] + desc 'Check replication source-> replica works on' + set +o xtrace ----------------------------------------------------------------------------------- Check replication source-> replica works on ----------------------------------------------------------------------------------- + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 10.131.241.25 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.131.241.25 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-2-80.sql ']' + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 10.131.241.25 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.131.241.25 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CsP0J9D8RH +++ mktemp ++ local LAST_ERR=/tmp/tmp.RA96Q8RtYR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.CsP0J9D8RH ++ cat /tmp/tmp.RA96Q8RtYR ++ rm /tmp/tmp.CsP0J9D8RH /tmp/tmp.RA96Q8RtYR ++ return 0 + client_pod=pxc-client-5d749ff8b6-5t9cj + wait_pod pxc-client-5d749ff8b6-5t9cj + local pod=pxc-client-5d749ff8b6-5t9cj + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-5t9cj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-5t9cj.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.K32pxtTtiP/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-2.sql /tmp/tmp.K32pxtTtiP/select-2.sql + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 10.131.252.27 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.131.252.27 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-2-80.sql ']' + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 10.131.252.27 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.131.252.27 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wVQhaFCRqN +++ mktemp ++ local LAST_ERR=/tmp/tmp.tL9Mt13Ivw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.wVQhaFCRqN ++ cat /tmp/tmp.tL9Mt13Ivw ++ rm /tmp/tmp.wVQhaFCRqN /tmp/tmp.tL9Mt13Ivw ++ return 0 + client_pod=pxc-client-5d749ff8b6-5t9cj + wait_pod pxc-client-5d749ff8b6-5t9cj + local pod=pxc-client-5d749ff8b6-5t9cj + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-5t9cj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-5t9cj.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.K32pxtTtiP/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-2.sql /tmp/tmp.K32pxtTtiP/select-2.sql + compare_mysql_cmd select-2 'SELECT * from myApp.testSourceReplica;' '-h 10.131.245.142 -uroot -proot_password' + local command_id=select-2 + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.131.245.142 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-2.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-2-80.sql ']' + run_mysql 'SELECT * from myApp.testSourceReplica;' '-h 10.131.245.142 -uroot -proot_password' + local 'command=SELECT * from myApp.testSourceReplica;' + local 'uri=-h 10.131.245.142 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qQ3n4IGVGA +++ mktemp ++ local LAST_ERR=/tmp/tmp.Dhsss4zHzr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.qQ3n4IGVGA ++ cat /tmp/tmp.Dhsss4zHzr ++ rm /tmp/tmp.qQ3n4IGVGA /tmp/tmp.Dhsss4zHzr ++ return 0 + client_pod=pxc-client-5d749ff8b6-5t9cj + wait_pod pxc-client-5d749ff8b6-5t9cj + local pod=pxc-client-5d749ff8b6-5t9cj + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-5t9cj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-5t9cj.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.K32pxtTtiP/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-2.sql /tmp/tmp.K32pxtTtiP/select-2.sql + desc 'Switch clusters over' + set +o xtrace ----------------------------------------------------------------------------------- Switch clusters over ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.vDW2JEv1EZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.oxeBqhYe6Z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.vDW2JEv1EZ ++ cat /tmp/tmp.oxeBqhYe6Z ++ rm /tmp/tmp.vDW2JEv1EZ /tmp/tmp.oxeBqhYe6Z ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site --namespace=cross-site-10017 ++ mktemp + local LAST_OUT=/tmp/tmp.tIoodkjszO ++ mktemp + local LAST_ERR=/tmp/tmp.cjQNAhJYfG + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site --namespace=cross-site-10017 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.tIoodkjszO Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site" modified. + cat /tmp/tmp.cjQNAhJYfG + rm /tmp/tmp.tIoodkjszO /tmp/tmp.cjQNAhJYfG + return 0 + run_mysql 'DELETE FROM myApp.myApp WHERE id=100500' '-h 10.131.251.87 -uroot -proot_password' + local 'command=DELETE FROM myApp.myApp WHERE id=100500' + local 'uri=-h 10.131.251.87 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.u697BF97TI +++ mktemp ++ local LAST_ERR=/tmp/tmp.sR7tdTqi4N ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.u697BF97TI ++ cat /tmp/tmp.sR7tdTqi4N ++ rm /tmp/tmp.u697BF97TI /tmp/tmp.sR7tdTqi4N ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-vjk86 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + run_mysql 'DELETE FROM myApp.testSourceReplica WHERE id=100700' '-h 10.131.251.87 -uroot -proot_password' + local 'command=DELETE FROM myApp.testSourceReplica WHERE id=100700' + local 'uri=-h 10.131.251.87 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V8iDmuQ1yS +++ mktemp ++ local LAST_ERR=/tmp/tmp.p14yqcFwSW ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.V8iDmuQ1yS ++ cat /tmp/tmp.p14yqcFwSW ++ rm /tmp/tmp.V8iDmuQ1yS /tmp/tmp.p14yqcFwSW ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-vjk86 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + kubectl_bin patch pxc cross-site-source --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": false, "sourcesList": [{"host": "10.131.241.25", "port": 3306, "weight": 100},{"host": "10.131.252.27", "port": 3306, "weight": 100},{"host": "10.131.245.142", "port": 3306, "weight": 100}]}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.XDtnZrEop2 ++ mktemp + local LAST_ERR=/tmp/tmp.JJ4Lta8qzz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch pxc cross-site-source --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": false, "sourcesList": [{"host": "10.131.241.25", "port": 3306, "weight": 100},{"host": "10.131.252.27", "port": 3306, "weight": 100},{"host": "10.131.245.142", "port": 3306, "weight": 100}]}]}}}' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.XDtnZrEop2 perconaxtradbcluster.pxc.percona.com/cross-site-source patched + cat /tmp/tmp.JJ4Lta8qzz + rm /tmp/tmp.XDtnZrEop2 /tmp/tmp.JJ4Lta8qzz + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.fA5dZu94nu +++ mktemp ++ local LAST_ERR=/tmp/tmp.nnHxbnj7HQ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.fA5dZu94nu ++ cat /tmp/tmp.nnHxbnj7HQ ++ rm /tmp/tmp.fA5dZu94nu /tmp/tmp.nnHxbnj7HQ ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site --namespace=cross-site-replica-27981 ++ mktemp + local LAST_OUT=/tmp/tmp.xtvTFRsjIW ++ mktemp + local LAST_ERR=/tmp/tmp.KetqvbB1on + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site --namespace=cross-site-replica-27981 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.xtvTFRsjIW Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site" modified. + cat /tmp/tmp.KetqvbB1on + rm /tmp/tmp.xtvTFRsjIW /tmp/tmp.KetqvbB1on + return 0 + kubectl_bin patch pxc cross-site-replica --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": true, "sourcesList": [{"host": "10.131.251.87", "port": 3306, "weight": 100},{"host": "10.131.243.40", "port": 3306, "weight": 100},{"host": "10.131.240.192", "port": 3306, "weight": 100}]}]}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.l8eRR84TS5 ++ mktemp + local LAST_ERR=/tmp/tmp.ONFzfsBjkY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl patch pxc cross-site-replica --type=merge --patch '{"spec": {"pxc": {"replicationChannels":[{"name": "source_to_replica", "isSource": true, "sourcesList": [{"host": "10.131.251.87", "port": 3306, "weight": 100},{"host": "10.131.243.40", "port": 3306, "weight": 100},{"host": "10.131.240.192", "port": 3306, "weight": 100}]}]}}}' + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.l8eRR84TS5 perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + cat /tmp/tmp.ONFzfsBjkY + rm /tmp/tmp.l8eRR84TS5 /tmp/tmp.ONFzfsBjkY + return 0 + desc 'Write data to replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- Write data to replica cluster ----------------------------------------------------------------------------------- + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testReplicaSource (id int PRIMARY KEY);' '-h 10.131.241.25 -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS testReplicaSource (id int PRIMARY KEY);' + local 'uri=-h 10.131.241.25 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2I9nNHPl9Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.V3j76lVNIS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.2I9nNHPl9Q ++ cat /tmp/tmp.V3j76lVNIS ++ rm /tmp/tmp.2I9nNHPl9Q /tmp/tmp.V3j76lVNIS ++ return 0 + client_pod=pxc-client-5d749ff8b6-5t9cj + wait_pod pxc-client-5d749ff8b6-5t9cj + local pod=pxc-client-5d749ff8b6-5t9cj + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-5t9cj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-5t9cj.Ok + set +o xtrace + run_mysql 'INSERT myApp.testReplicaSource (id) VALUES (100800)' '-h 10.131.241.25 -uroot -proot_password' + local 'command=INSERT myApp.testReplicaSource (id) VALUES (100800)' + local 'uri=-h 10.131.241.25 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FZWXEmtCN6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.IIWL8tOmJ4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.FZWXEmtCN6 ++ cat /tmp/tmp.IIWL8tOmJ4 ++ rm /tmp/tmp.FZWXEmtCN6 /tmp/tmp.IIWL8tOmJ4 ++ return 0 + client_pod=pxc-client-5d749ff8b6-5t9cj + wait_pod pxc-client-5d749ff8b6-5t9cj + local pod=pxc-client-5d749ff8b6-5t9cj + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-5t9cj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-5t9cj.Ok + set +o xtrace + sleep 15 + desc 'Check replication replica -> source works on' + set +o xtrace ----------------------------------------------------------------------------------- Check replication replica -> source works on ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.VFDgRNBWTH +++ mktemp ++ local LAST_ERR=/tmp/tmp.hfj4qLRmwR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.VFDgRNBWTH ++ cat /tmp/tmp.hfj4qLRmwR ++ rm /tmp/tmp.VFDgRNBWTH /tmp/tmp.hfj4qLRmwR ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site --namespace=cross-site-10017 ++ mktemp + local LAST_OUT=/tmp/tmp.0pYRDBaXHl ++ mktemp + local LAST_ERR=/tmp/tmp.vDjJnuCD6s + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site --namespace=cross-site-10017 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.0pYRDBaXHl Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-cross-site" modified. + cat /tmp/tmp.vDjJnuCD6s + rm /tmp/tmp.0pYRDBaXHl /tmp/tmp.vDjJnuCD6s + return 0 + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 10.131.251.87 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.131.251.87 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-3-80.sql ']' + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 10.131.251.87 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.131.251.87 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7ZYoJDKviD +++ mktemp ++ local LAST_ERR=/tmp/tmp.rks20fNryT ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.7ZYoJDKviD ++ cat /tmp/tmp.rks20fNryT ++ rm /tmp/tmp.7ZYoJDKviD /tmp/tmp.rks20fNryT ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-vjk86 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.K32pxtTtiP/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-3.sql /tmp/tmp.K32pxtTtiP/select-3.sql + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 10.131.243.40 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.131.243.40 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-3-80.sql ']' + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 10.131.243.40 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.131.243.40 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pk1Mj4zTbD +++ mktemp ++ local LAST_ERR=/tmp/tmp.EqXhSHZKTq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Pk1Mj4zTbD ++ cat /tmp/tmp.EqXhSHZKTq ++ rm /tmp/tmp.Pk1Mj4zTbD /tmp/tmp.EqXhSHZKTq ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-vjk86 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.K32pxtTtiP/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-3.sql /tmp/tmp.K32pxtTtiP/select-3.sql + compare_mysql_cmd select-3 'SELECT * from myApp.testReplicaSource;' '-h 10.131.240.192 -uroot -proot_password' + local command_id=select-3 + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.131.240.192 -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-3.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-3-80.sql ']' + run_mysql 'SELECT * from myApp.testReplicaSource;' '-h 10.131.240.192 -uroot -proot_password' + local 'command=SELECT * from myApp.testReplicaSource;' + local 'uri=-h 10.131.240.192 -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8u76tKH4q4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ObQhddFHl6 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.8u76tKH4q4 ++ cat /tmp/tmp.ObQhddFHl6 ++ rm /tmp/tmp.8u76tKH4q4 /tmp/tmp.ObQhddFHl6 ++ return 0 + client_pod=pxc-client-5d749ff8b6-vjk86 + wait_pod pxc-client-5d749ff8b6-vjk86 + local pod=pxc-client-5d749ff8b6-vjk86 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-5d749ff8b6-vjk86 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-5d749ff8b6-vjk86.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.K32pxtTtiP/select-3.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/cross-site/compare/select-3.sql /tmp/tmp.K32pxtTtiP/select-3.sql + desc Success + set +o xtrace ----------------------------------------------------------------------------------- Success ----------------------------------------------------------------------------------- + destroy cross-site-10017 true + local namespace=cross-site-10017 + local ignore_logs=true + [[ true == \f\a\l\s\e ]] + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide + kubectl patch pxc -n cross-site-10017 cross-site-source --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + kubectl patch pxc -n cross-site-22346 cross-site-source --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + kubectl patch pxc -n cross-site-5864 cross-site-source --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-source patched + kubectl patch pxc -n cross-site-replica-1754 cross-site-replica --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + kubectl patch pxc -n cross-site-replica-27981 cross-site-replica --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + kubectl patch pxc -n cross-site-replica-31053 cross-site-replica --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/cross-site-replica patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.mLsnwBuYd3 ++ mktemp + local LAST_ERR=/tmp/tmp.J0FklbbmxE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.mLsnwBuYd3 perconaxtradbcluster.pxc.percona.com "cross-site-source" deleted perconaxtradbcluster.pxc.percona.com "cross-site-replica" deleted + cat /tmp/tmp.J0FklbbmxE + rm /tmp/tmp.mLsnwBuYd3 /tmp/tmp.J0FklbbmxE + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.TYswQ825vM ++ mktemp + local LAST_ERR=/tmp/tmp.nBvUEQTL2G + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.TYswQ825vM No resources found + cat /tmp/tmp.nBvUEQTL2G + rm /tmp/tmp.TYswQ825vM /tmp/tmp.nBvUEQTL2G + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.iW56NOsHUp ++ mktemp + local LAST_ERR=/tmp/tmp.hyigd9eNqx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.iW56NOsHUp No resources found + cat /tmp/tmp.hyigd9eNqx + rm /tmp/tmp.iW56NOsHUp /tmp/tmp.hyigd9eNqx + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.3KL59erySc ++ mktemp + local LAST_ERR=/tmp/tmp.uYDx3VG1kb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + cat /tmp/tmp.3KL59erySc + cat /tmp/tmp.uYDx3VG1kb Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "percona-xtradbcluster-webhook" not found + rm /tmp/tmp.3KL59erySc /tmp/tmp.uYDx3VG1kb + return 1 + : + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n '' ']' + rm -rf /tmp/tmp.K32pxtTtiP + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-10017 ++ mktemp + destroy cross-site-replica-27981 true + local namespace=cross-site-replica-27981 + local ignore_logs=true + [[ true == \f\a\l\s\e ]] + local LAST_OUT=/tmp/tmp.HREPPNfSmb + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' ++ mktemp + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide + local LAST_ERR=/tmp/tmp.rdvKIGO3eW + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete --grace-period=0 --force=true namespace cross-site-10017 No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name, label selector, or --all flag specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.0awsnQUSmu ++ mktemp + local LAST_ERR=/tmp/tmp.wdGEwZBU7j + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.0awsnQUSmu No resources found + cat /tmp/tmp.wdGEwZBU7j + rm /tmp/tmp.0awsnQUSmu /tmp/tmp.wdGEwZBU7j + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.tG3nMi0Ng0 ++ mktemp + local LAST_ERR=/tmp/tmp.FdkYSQk0qn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.tG3nMi0Ng0 No resources found + cat /tmp/tmp.FdkYSQk0qn + rm /tmp/tmp.tG3nMi0Ng0 /tmp/tmp.FdkYSQk0qn + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.QDXdw5dNvJ ++ mktemp + local LAST_ERR=/tmp/tmp.AQzmNAnKyY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.QDXdw5dNvJ No resources found + cat /tmp/tmp.AQzmNAnKyY + rm /tmp/tmp.QDXdw5dNvJ /tmp/tmp.AQzmNAnKyY + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.4nAnJakzow ++ mktemp + local LAST_ERR=/tmp/tmp.9Nu5ipL4Sb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + cat /tmp/tmp.4nAnJakzow + cat /tmp/tmp.9Nu5ipL4Sb Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "percona-xtradbcluster-webhook" not found + rm /tmp/tmp.4nAnJakzow /tmp/tmp.9Nu5ipL4Sb + return 1 + : + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n '' ']' + rm -rf /tmp/tmp.K32pxtTtiP + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-replica-27981 ++ mktemp + local LAST_OUT=/tmp/tmp.pmchC0E1Vm ++ mktemp + local LAST_ERR=/tmp/tmp.DkF5auCBrX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete --grace-period=0 --force=true namespace cross-site-replica-27981