++ echo 'Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/logs/big-data.log' Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/logs/big-data.log ++ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP= ++ oc get projects error: No Auth Provider found for name "gcp" +++ grep '\-eks\-' +++ jq -r .serverVersion.gitVersion +++ kubectl version -o json ++ '[' ']' ++ EKS=0 +++ /usr/bin/sed -r 's/[^0-9.]+//g' +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ kubectl version -o json ++ KUBE_VERSION=1.20 +++ helm version -c +++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/' ++ HELM_VERSION=v3.8.1 ++ '[' v3 == v2 ']' + main + '[' -n '' ']' + create_infra big-data-29633 + local ns=big-data-29633 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "pxc" + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "pxc" + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.Id8FqJkz1w ++ mktemp + local LAST_ERR=/tmp/tmp.oNfODP9x0n + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc --all --all-namespaces + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete pxc --all --all-namespaces + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete pxc --all --all-namespaces + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + cat /tmp/tmp.Id8FqJkz1w + cat /tmp/tmp.oNfODP9x0n error: the server doesn't have a resource type "pxc" + rm /tmp/tmp.Id8FqJkz1w /tmp/tmp.oNfODP9x0n + return 1 + : + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.0MBu81HHA1 ++ mktemp + local LAST_ERR=/tmp/tmp.9qSY3nxJvt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-backup --all --all-namespaces + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete pxc-backup --all --all-namespaces + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete pxc-backup --all --all-namespaces + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + cat /tmp/tmp.0MBu81HHA1 + cat /tmp/tmp.9qSY3nxJvt error: the server doesn't have a resource type "pxc-backup" + rm /tmp/tmp.0MBu81HHA1 /tmp/tmp.9qSY3nxJvt + return 1 + : + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.iQs4QTLwRk ++ mktemp + local LAST_ERR=/tmp/tmp.m8FiNvUybM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-restore --all --all-namespaces + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete pxc-restore --all --all-namespaces + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete pxc-restore --all --all-namespaces + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + cat /tmp/tmp.iQs4QTLwRk + cat /tmp/tmp.m8FiNvUybM error: the server doesn't have a resource type "pxc-restore" + rm /tmp/tmp.iQs4QTLwRk /tmp/tmp.m8FiNvUybM + return 1 + : + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + '[' '!' -z '' ']' + kubectl_bin delete namespace pxc-operator + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^NAME' + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.Nq15B25WYJ ++ mktemp + local LAST_ERR=/tmp/tmp.DKHyXxam9M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl get ns + awk '{print$1}' + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.5t0dZeqio1 ++ mktemp + local LAST_ERR=/tmp/tmp.a0kTg8pKao + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete namespace pxc-operator + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.Nq15B25WYJ + cat /tmp/tmp.DKHyXxam9M + rm /tmp/tmp.Nq15B25WYJ /tmp/tmp.DKHyXxam9M + return 0 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace pxc-operator error: resource(s) were provided, but no name, label selector, or --all flag specified + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace pxc-operator + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + cat /tmp/tmp.5t0dZeqio1 + cat /tmp/tmp.a0kTg8pKao Error from server (NotFound): namespaces "pxc-operator" not found + rm /tmp/tmp.5t0dZeqio1 /tmp/tmp.a0kTg8pKao + return 1 + : + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + set +o xtrace namespace/pxc-operator - Error from server (NotFound): namespaces "pxc-operator" not found + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.xA5pe9bZju ++ mktemp + local LAST_ERR=/tmp/tmp.2WcrcNYxxv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create namespace pxc-operator + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.xA5pe9bZju namespace/pxc-operator created + cat /tmp/tmp.2WcrcNYxxv + rm /tmp/tmp.xA5pe9bZju /tmp/tmp.2WcrcNYxxv + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.pBO2U1zyU1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.hVB8IidMiA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.pBO2U1zyU1 ++ cat /tmp/tmp.hVB8IidMiA ++ rm /tmp/tmp.pBO2U1zyU1 /tmp/tmp.hVB8IidMiA ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-bigdata --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.olzHWDcuwx ++ mktemp + local LAST_ERR=/tmp/tmp.DS4UlbCd6M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-bigdata --namespace=pxc-operator + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.olzHWDcuwx Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-bigdata" modified. + cat /tmp/tmp.DS4UlbCd6M + rm /tmp/tmp.olzHWDcuwx /tmp/tmp.DS4UlbCd6M + return 0 + deploy_operator + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.EX3pDdgEG0 ++ mktemp + local LAST_ERR=/tmp/tmp.2RH4zZye5n + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/crd.yaml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.EX3pDdgEG0 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com created customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com created customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com created customresourcedefinition.apiextensions.k8s.io/perconaxtradbbackups.pxc.percona.com created + cat /tmp/tmp.2RH4zZye5n + rm /tmp/tmp.EX3pDdgEG0 /tmp/tmp.2RH4zZye5n + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.NhMjFu4Ijh ++ mktemp + local LAST_ERR=/tmp/tmp.9R7LJV5HAk + local exit_status=0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/cw-rbac.yaml ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.NhMjFu4Ijh clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator created serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator created + cat /tmp/tmp.9R7LJV5HAk + rm /tmp/tmp.NhMjFu4Ijh /tmp/tmp.9R7LJV5HAk + return 0 + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/cw-operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.mFN3avT5aN ++ mktemp + local LAST_ERR=/tmp/tmp.lHKqyZsA2g + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.mFN3avT5aN deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.lHKqyZsA2g + rm /tmp/tmp.mFN3avT5aN /tmp/tmp.lHKqyZsA2g + return 0 + sleep 10 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Zy90Fyc18i +++ mktemp ++ local LAST_ERR=/tmp/tmp.iN0AqXod0u ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.Zy90Fyc18i ++ cat /tmp/tmp.iN0AqXod0u ++ rm /tmp/tmp.Zy90Fyc18i /tmp/tmp.iN0AqXod0u ++ return 0 + wait_pod percona-xtradb-cluster-operator-5699d7755d-d9rn4 480 pxc-operator + local pod=percona-xtradb-cluster-operator-5699d7755d-d9rn4 + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-5699d7755d-d9rn4 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace percona-xtradb-cluster-operator-5699d7755d-d9rn4.Ok + sleep 3 + create_namespace big-data-29633 + local namespace=big-data-29633 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + '[' '!' -z '' ']' + kubectl_bin delete namespace big-data-29633 ++ mktemp + xargs kubectl delete ns + awk '{print$1}' + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^NAME' + local LAST_OUT=/tmp/tmp.mzlaFCEi9t + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.zp0j05TWwK ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.tkLaa2uZgv + local LAST_ERR=/tmp/tmp.a77hXyHndQ + local exit_status=0 + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl get ns + for i in '$(seq 0 2)' + kubectl delete namespace big-data-29633 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace big-data-29633 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.zp0j05TWwK + cat /tmp/tmp.tkLaa2uZgv + rm /tmp/tmp.zp0j05TWwK /tmp/tmp.tkLaa2uZgv + return 0 error: resource(s) were provided, but no name, label selector, or --all flag specified + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace big-data-29633 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + cat /tmp/tmp.mzlaFCEi9t + cat /tmp/tmp.a77hXyHndQ Error from server (NotFound): namespaces "big-data-29633" not found + rm /tmp/tmp.mzlaFCEi9t /tmp/tmp.a77hXyHndQ + return 1 + : + wait_for_delete namespace/big-data-29633 + local res=namespace/big-data-29633 + set +o xtrace namespace/big-data-29633 - Error from server (NotFound): namespaces "big-data-29633" not found + kubectl_bin create namespace big-data-29633 ++ mktemp + local LAST_OUT=/tmp/tmp.cXAJCodTwz ++ mktemp + local LAST_ERR=/tmp/tmp.nL79F8c7tw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create namespace big-data-29633 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.cXAJCodTwz namespace/big-data-29633 created + cat /tmp/tmp.nL79F8c7tw + rm /tmp/tmp.cXAJCodTwz /tmp/tmp.nL79F8c7tw + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.T1n6fUmEoG +++ mktemp ++ local LAST_ERR=/tmp/tmp.yIXr6yZran ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.T1n6fUmEoG ++ cat /tmp/tmp.yIXr6yZran ++ rm /tmp/tmp.T1n6fUmEoG /tmp/tmp.yIXr6yZran ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-bigdata --namespace=big-data-29633 ++ mktemp + local LAST_OUT=/tmp/tmp.qpqenngoM5 ++ mktemp + local LAST_ERR=/tmp/tmp.pWQdaTGE2W + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-bigdata --namespace=big-data-29633 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.qpqenngoM5 Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-bigdata" modified. + cat /tmp/tmp.pWQdaTGE2W + rm /tmp/tmp.qpqenngoM5 /tmp/tmp.pWQdaTGE2W + return 0 + apply_secrets + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.QoOQs5vp1P ++ mktemp + local LAST_ERR=/tmp/tmp.UCbWp5wzEg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.QoOQs5vp1P secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.UCbWp5wzEg + rm /tmp/tmp.QoOQs5vp1P /tmp/tmp.UCbWp5wzEg + return 0 + deploy_cert_manager + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.844jAL3mPO ++ mktemp + local LAST_ERR=/tmp/tmp.44efAwxyaQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create namespace cert-manager + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.844jAL3mPO namespace/cert-manager created + cat /tmp/tmp.44efAwxyaQ + rm /tmp/tmp.844jAL3mPO /tmp/tmp.44efAwxyaQ + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.0xNoyyZ0ME ++ mktemp + local LAST_ERR=/tmp/tmp.sJu1royeM7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.0xNoyyZ0ME namespace/cert-manager labeled + cat /tmp/tmp.sJu1royeM7 + rm /tmp/tmp.0xNoyyZ0ME /tmp/tmp.sJu1royeM7 + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.gHPb7HTU1P ++ mktemp + local LAST_ERR=/tmp/tmp.p0cEach2V0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml --validate=false + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.gHPb7HTU1P customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io created namespace/cert-manager configured serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrole.rbac.authorization.k8s.io/cert-manager-view created clusterrole.rbac.authorization.k8s.io/cert-manager-edit created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created role.rbac.authorization.k8s.io/cert-manager:leaderelection created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created + cat /tmp/tmp.p0cEach2V0 Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.gHPb7HTU1P /tmp/tmp.p0cEach2V0 + return 0 + sleep 60 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + cluster=some-name + spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/conf/some-name.yml 3 10 /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets_without_tls.yml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/conf/some-name.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets_without_tls.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets_without_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Ip3v002LBo ++ mktemp + local LAST_ERR=/tmp/tmp.xirtafQF84 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/secrets_without_tls.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.Ip3v002LBo secret/my-cluster-secrets created + cat /tmp/tmp.xirtafQF84 + rm /tmp/tmp.Ip3v002LBo /tmp/tmp.xirtafQF84 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1-11-0#' + local LAST_OUT=/tmp/tmp.0s5HuKFrvB + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.big-data-29633~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.7E0n7Lp6m7 + local exit_status=0 + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.0s5HuKFrvB deployment.apps/pxc-client created + cat /tmp/tmp.7E0n7Lp6m7 + rm /tmp/tmp.0s5HuKFrvB /tmp/tmp.7E0n7Lp6m7 + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/conf/some-name.yml + kubectl_bin apply -f - ++ mktemp + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/conf/some-name.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1-11-0#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.EvyqS7q89u + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#initImage:.*-init$#initImage: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.big-data-29633~ ++ mktemp + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + local LAST_ERR=/tmp/tmp.LogOi6QLqR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.EvyqS7q89u perconaxtradbcluster.pxc.percona.com/some-name created + cat /tmp/tmp.LogOi6QLqR + rm /tmp/tmp.EvyqS7q89u /tmp/tmp.LogOi6QLqR + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oYoHlVqPyT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZdZWo6kJ1w +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.oYoHlVqPyT +++ cat /tmp/tmp.ZdZWo6kJ1w +++ rm /tmp/tmp.oYoHlVqPyT /tmp/tmp.ZdZWo6kJ1w +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.L70OAoep3M ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FezfAnDqcf +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ cat /tmp/tmp.L70OAoep3M +++ cat /tmp/tmp.FezfAnDqcf +++ rm /tmp/tmp.L70OAoep3M /tmp/tmp.FezfAnDqcf +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo some-name-proxysql ++ return + local proxy=some-name-proxysql + wait_for_running some-name-proxysql 1 + local name=some-name-proxysql + let last_pod=0 + : + local max_retry=480 ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-proxysql-0 480 + local pod=some-name-proxysql-0 + local max_retry=480 + local ns= ++ echo some-name-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace some-name-proxysql-0............Ok + wait_for_running some-name-pxc 3 + local name=some-name-pxc + let last_pod=2 + local max_retry=480 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-pxc-0 + local container=pxc + set +o xtrace some-name-pxc-0....................................Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-1 480 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace some-name-pxc-1............................................Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-2 480 + local pod=some-name-pxc-2 + local max_retry=480 + local ns= ++ echo some-name-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace some-name-pxc-2...................................Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-proxysql -uroot -proot_password' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h some-name-proxysql -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OBcjxTuYth +++ mktemp ++ local LAST_ERR=/tmp/tmp.JpKS4m5ku5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.OBcjxTuYth ++ cat /tmp/tmp.JpKS4m5ku5 ++ rm /tmp/tmp.OBcjxTuYth /tmp/tmp.JpKS4m5ku5 ++ return 0 + client_pod=pxc-client-5d749ff8b6-ld4nq + wait_pod pxc-client-5d749ff8b6-ld4nq + local pod=pxc-client-5d749ff8b6-ld4nq + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-5d749ff8b6-ld4nq + local container= + set +o xtrace pxc-client-5d749ff8b6-ld4nq.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-proxysql -uroot -proot_password' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h some-name-proxysql -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oiD14H6OTv +++ mktemp ++ local LAST_ERR=/tmp/tmp.JVgv0oJv03 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.oiD14H6OTv ++ cat /tmp/tmp.JVgv0oJv03 ++ rm /tmp/tmp.oiD14H6OTv /tmp/tmp.JVgv0oJv03 ++ return 0 + client_pod=pxc-client-5d749ff8b6-ld4nq + wait_pod pxc-client-5d749ff8b6-ld4nq + local pod=pxc-client-5d749ff8b6-ld4nq + local max_retry=480 + local ns= ++ echo pxc-client-5d749ff8b6-ld4nq ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pxc-client-5d749ff8b6-ld4nq.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VpSeft0fsd +++ mktemp ++ local LAST_ERR=/tmp/tmp.nDpxxOCQnI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.VpSeft0fsd ++ cat /tmp/tmp.nDpxxOCQnI ++ rm /tmp/tmp.VpSeft0fsd /tmp/tmp.nDpxxOCQnI ++ return 0 + client_pod=pxc-client-5d749ff8b6-ld4nq + wait_pod pxc-client-5d749ff8b6-ld4nq + local pod=pxc-client-5d749ff8b6-ld4nq + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-ld4nq + local container= + set +o xtrace pxc-client-5d749ff8b6-ld4nq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qbBlUGG89M/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/compare/select-1.sql /tmp/tmp.qbBlUGG89M/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8Hfpp2q2U5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4FGlkF2yUy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.8Hfpp2q2U5 ++ cat /tmp/tmp.4FGlkF2yUy ++ rm /tmp/tmp.8Hfpp2q2U5 /tmp/tmp.4FGlkF2yUy ++ return 0 + client_pod=pxc-client-5d749ff8b6-ld4nq + wait_pod pxc-client-5d749ff8b6-ld4nq + local pod=pxc-client-5d749ff8b6-ld4nq + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-ld4nq + local container= + set +o xtrace pxc-client-5d749ff8b6-ld4nq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qbBlUGG89M/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/compare/select-1.sql /tmp/tmp.qbBlUGG89M/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tdPpYAeHLv +++ mktemp ++ local LAST_ERR=/tmp/tmp.iIAHD14QTB ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.tdPpYAeHLv ++ cat /tmp/tmp.iIAHD14QTB ++ rm /tmp/tmp.tdPpYAeHLv /tmp/tmp.iIAHD14QTB ++ return 0 + client_pod=pxc-client-5d749ff8b6-ld4nq + wait_pod pxc-client-5d749ff8b6-ld4nq + local pod=pxc-client-5d749ff8b6-ld4nq + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-5d749ff8b6-ld4nq + local container= + set +o xtrace pxc-client-5d749ff8b6-ld4nq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qbBlUGG89M/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/compare/select-1.sql /tmp/tmp.qbBlUGG89M/select-1.sql ++ is_keyring_plugin_in_use some-name ++ local cluster=some-name ++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KO7ZwOicwI +++ mktemp ++ local LAST_ERR=/tmp/tmp.KL9mhIIOSt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.KO7ZwOicwI ++ cat /tmp/tmp.KL9mhIIOSt Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.KO7ZwOicwI /tmp/tmp.KL9mhIIOSt ++ return 0 + '[' '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/conf/crd_backup.yml ++ mktemp + local LAST_OUT=/tmp/tmp.rA5OysRUsg ++ mktemp + local LAST_ERR=/tmp/tmp.Z3pv0fdtWS + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/conf/crd_backup.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.rA5OysRUsg customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com configured + cat /tmp/tmp.Z3pv0fdtWS + rm /tmp/tmp.rA5OysRUsg /tmp/tmp.Z3pv0fdtWS + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + desc 'Switch to 8.0 backup' + set +o xtrace ----------------------------------------------------------------------------------- Switch to 8.0 backup ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/conf/backup.yml ++ mktemp + local LAST_OUT=/tmp/tmp.VOGT7Z6Lt2 ++ mktemp + local LAST_ERR=/tmp/tmp.aDNMT8vSiM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/conf/backup.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.VOGT7Z6Lt2 perconaxtradbclusterbackup.pxc.percona.com/backup1 created + cat /tmp/tmp.aDNMT8vSiM + rm /tmp/tmp.VOGT7Z6Lt2 /tmp/tmp.aDNMT8vSiM + return 0 + sleep 10 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/conf/restore-backup.yml ++ mktemp + local LAST_OUT=/tmp/tmp.6ON2SXM91y ++ mktemp + local LAST_ERR=/tmp/tmp.3XsKjUgbKe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/big-data/conf/restore-backup.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.6ON2SXM91y perconaxtradbclusterrestore.pxc.percona.com/restore1 created + cat /tmp/tmp.3XsKjUgbKe + rm /tmp/tmp.6ON2SXM91y /tmp/tmp.3XsKjUgbKe + return 0 + wait_backup_restore restore1 + local backup_name=restore1 + set +o xtrace restore1.............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Succeeded + wait_for_running some-name-proxysql 1 + local name=some-name-proxysql + let last_pod=0 + : + local max_retry=480 ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-proxysql-0 480 + local pod=some-name-proxysql-0 + local max_retry=480 + local ns= ++ echo some-name-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace some-name-proxysql-0.Ok + wait_for_running some-name-pxc 3 + local name=some-name-pxc + let last_pod=2 + local max_retry=480 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-pxc-0 + local container=pxc + set +o xtrace some-name-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-1 480 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ echo some-name-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace some-name-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-2 480 + local pod=some-name-pxc-2 + local max_retry=480 + local ns= ++ echo some-name-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace some-name-pxc-2.Ok + kubectl_bin delete pvc/datadir-some-name-pxc-2 pod/some-name-pxc-2 ++ mktemp + local LAST_OUT=/tmp/tmp.Kxp3tgJH5V ++ mktemp + local LAST_ERR=/tmp/tmp.aujMyGyYor + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pvc/datadir-some-name-pxc-2 pod/some-name-pxc-2 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.Kxp3tgJH5V persistentvolumeclaim "datadir-some-name-pxc-2" deleted pod "some-name-pxc-2" deleted + cat /tmp/tmp.aujMyGyYor + rm /tmp/tmp.Kxp3tgJH5V /tmp/tmp.aujMyGyYor + return 0 + sleep 1 + kubectl_bin delete pod/some-name-pxc-2 ++ mktemp + local LAST_OUT=/tmp/tmp.4m2OA7hSnc ++ mktemp + local LAST_ERR=/tmp/tmp.PkWmHLt2ET + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pod/some-name-pxc-2 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.4m2OA7hSnc pod "some-name-pxc-2" deleted + cat /tmp/tmp.PkWmHLt2ET + rm /tmp/tmp.4m2OA7hSnc /tmp/tmp.PkWmHLt2ET + return 0 + wait_for_running some-name-pxc 3 800 + local name=some-name-pxc + let last_pod=2 + local max_retry=800 ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 800 + local pod=some-name-pxc-0 + local max_retry=800 + local ns= ++ echo some-name-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace some-name-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-1 800 + local pod=some-name-pxc-1 + local max_retry=800 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-pxc-1 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace some-name-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-2 800 + local pod=some-name-pxc-2 + local max_retry=800 + local ns= ++ echo some-name-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace some-name-pxc-2.................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Ok + destroy big-data-29633 + local namespace=big-data-29633 + local ignore_logs=false + [[ false == \f\a\l\s\e ]] ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ + tee /tmp/tmp.qbBlUGG89M/operator.log + grep -v level=info + grep -v 'the object has been modified' + sort -u + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator + grep -v 'get backup status: Job.batch' ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.rD15CHSHUb +++ mktemp ++ local LAST_ERR=/tmp/tmp.rM7Ca8LgSH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.rD15CHSHUb ++ cat /tmp/tmp.rM7Ca8LgSH ++ rm /tmp/tmp.rD15CHSHUb /tmp/tmp.rM7Ca8LgSH ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-5699d7755d-d9rn4 ++ mktemp + local LAST_OUT=/tmp/tmp.2ylVXOouVl ++ mktemp + local LAST_ERR=/tmp/tmp.ukWBRnMBui + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-5699d7755d-d9rn4 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.2ylVXOouVl + cat /tmp/tmp.ukWBRnMBui + rm /tmp/tmp.2ylVXOouVl /tmp/tmp.ukWBRnMBui + return 0 I0412 09:20:47.779751 1 request.go:645] Throttling request took 1.045608983s, request: GET:https://10.31.240.1:443/apis/networking.gke.io/v1?timeout=32s I0412 09:22:41.630756 1 request.go:645] Throttling request took 1.042604231s, request: GET:https://10.31.240.1:443/apis/rbac.authorization.k8s.io/v1beta1?timeout=32s {"level":"error",,"caller":"pxc/controller.go:1142","msg":"sync users","error":"exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.big-data-29633.s' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.big-data-29633.s' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.big-data-29633.s' (using password: YES)\nERROR (line:1921) : Could not find a primary cluster node\n","errorVerbose":"exec syncusers: command terminated with exit code 1 / / ERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.big-data-29633.s' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.big-data-29633.s' (using password: YES)\nERROR 1045 (28000): Access denied for user 'operator'@'some-name-proxysql-0.some-name-proxysql-unready.big-data-29633.s' (using password: YES)\nERROR (line:1921) : Could not find a primary cluster node\n\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:491\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1140\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581","stacktrace":"github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1142"} {"level":"info",,"caller":"pxc/controller.go:468","msg":"reconcile replication error","err":"failed to connect to pod some-name-pxc-0: Error 1045: Access denied for user 'operator'@'10.28.1.6' (using password: YES)"} {"level":"info",,"caller":"pxc/controller.go:468","msg":"reconcile replication error","err":"get primary pxc pod: not found"} {"level":"info",,"caller":"pxcrestore/controller.go:121","msg":"backup restore request"} {"level":"info",,"caller":"pxcrestore/controller.go:181","msg":"stopping cluster","cluster":"some-name"} {"level":"info",,"caller":"pxcrestore/controller.go:193","msg":"starting restore","cluster":"some-name","backup":"backup1"} {"level":"info",,"caller":"pxcrestore/controller.go:205","msg":"starting cluster","cluster":"some-name"} {"level":"info",,"caller":"pxcrestore/controller.go:243","msg":"You can view xtrabackup log:\n$ kubectl logs job/restore-job-restore1-some-name\nIf everything is fine, you can cleanup the job:\n$ kubectl delete pxc-restore/restore1\n"} {"level":"info",,"caller":"pxc/version.go:328","msg":"update PXC version (fetched from db)","new version":"8.0.27-18.1"} {"level":"info",,"caller":"v1/pxc_types.go:874","msg":"ProxySQL size will be changed from 1 to 2 due to safe config"} {"level":"info",,"caller":"v1/pxc_types.go:875","msg":"Set allowUnsafeConfigurations=true to disable safe configuration"} {"level":"info",,"logger":"cmd","msg":"Git commit: 706f792ae47c369cb3556faff186b6873a8a247f Git branch: PR-1125-706f792a Build time: 2022-04-12T09:09:41Z"} {"level":"info",,"logger":"cmd","msg":"Go OS/Arch: linux/amd64"} {"level":"info",,"logger":"cmd","msg":"Go Version: go1.17.8"} {"level":"info",,"logger":"cmd","msg":"operator-sdk Version: v0.19.4"} {"level":"info",,"logger":"cmd","msg":"Registering Components."} {"level":"info",,"logger":"cmd","msg":"Runs on","platform":"kubernetes","version":"v1.20.15-gke.4100"} {"level":"info",,"logger":"cmd","msg":"Starting the Cmd."} {"level":"info",,"logger":"controller-runtime.certwatcher","msg":"Starting certificate watcher"} {"level":"info",,"logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate"} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbclusterbackup-controller","msg":"Starting Controller"} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbclusterbackup-controller","msg":"Starting EventSource","source":"kind source: /, Kind="} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbclusterbackup-controller","msg":"Starting workers","worker count":1} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbcluster-controller","msg":"Starting Controller"} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbcluster-controller","msg":"Starting EventSource","source":"kind source: /, Kind="} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbcluster-controller","msg":"Starting workers","worker count":1} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbclusterrestore-controller","msg":"Starting Controller"} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbclusterrestore-controller","msg":"Starting EventSource","source":"kind source: /, Kind="} {"level":"info",,"logger":"controller-runtime.manager.controller.perconaxtradbclusterrestore-controller","msg":"Starting workers","worker count":1} {"level":"info",,"logger":"controller-runtime.manager","msg":"starting metrics server","path":"/metrics"} {"level":"info",,"logger":"controller-runtime.metrics","msg":"metrics server is starting to listen","addr":":8080"} {"level":"info",,"logger":"controller-runtime.webhook","msg":"registering webhook","path":"/validate-percona-xtradbcluster"} {"level":"info",,"logger":"controller-runtime.webhook","msg":"serving webhook server","host":"","port":9443} {"level":"info",,"logger":"controller-runtime.webhook.webhooks","msg":"starting webhook server"} {"level":"info",,"logger":"leader","msg":"Became the leader."} {"level":"info",,"logger":"leader","msg":"No pre-existing lock was found."} {"level":"info",,"logger":"leader","msg":"Trying to become the leader."} + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n big-data-29633 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/some-name patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.dRxr7Ho8Jh ++ mktemp + local LAST_ERR=/tmp/tmp.hOsq1EXjeC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.dRxr7Ho8Jh perconaxtradbcluster.pxc.percona.com "some-name" deleted + cat /tmp/tmp.hOsq1EXjeC + rm /tmp/tmp.dRxr7Ho8Jh /tmp/tmp.hOsq1EXjeC + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.bYO5Bmjh1M ++ mktemp + local LAST_ERR=/tmp/tmp.ck2DDROEoi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.bYO5Bmjh1M perconaxtradbclusterbackup.pxc.percona.com "backup1" deleted + cat /tmp/tmp.ck2DDROEoi + rm /tmp/tmp.bYO5Bmjh1M /tmp/tmp.ck2DDROEoi + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.zD9XhvDeiP ++ mktemp + local LAST_ERR=/tmp/tmp.32GcCKEfVL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.zD9XhvDeiP perconaxtradbclusterrestore.pxc.percona.com "restore1" deleted + cat /tmp/tmp.32GcCKEfVL + rm /tmp/tmp.zD9XhvDeiP /tmp/tmp.32GcCKEfVL + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.GY4Mvgp74i ++ mktemp + local LAST_ERR=/tmp/tmp.AGzNcEVWmH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.GY4Mvgp74i validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.AGzNcEVWmH + rm /tmp/tmp.GY4Mvgp74i /tmp/tmp.AGzNcEVWmH + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml namespace "cert-manager" deleted + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + rm -rf /tmp/tmp.qbBlUGG89M + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator + kubectl_bin delete --grace-period=0 --force=true namespace big-data-29633 ++ mktemp + local LAST_OUT=/tmp/tmp.x1CKNJVo9R ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.xvKeqghW4I + local exit_status=0 + local LAST_OUT=/tmp/tmp.4AIOaCc51k ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp + local LAST_ERR=/tmp/tmp.iB2hhYD5sV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete --grace-period=0 --force=true namespace big-data-29633