Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/logs/restore-to-encrypted-cluster-8-0.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + main + create_infra restore-to-encrypted-cluster-27994 + local ns=restore-to-encrypted-cluster-27994 + '[' -n pxc-operator ']' + read -r pns name + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + [[ -z restore-to-encrypted-cluster-7078 ]] + [[ -z some-name ]] + kubectl_bin patch pxc -n restore-to-encrypted-cluster-7078 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.vYk8eEFfHM ++ mktemp + local LAST_ERR=/tmp/tmp.gzkji5A1ps + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc -n restore-to-encrypted-cluster-7078 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vYk8eEFfHM perconaxtradbcluster.pxc.percona.com/some-name patched + cat /tmp/tmp.gzkji5A1ps + rm /tmp/tmp.vYk8eEFfHM /tmp/tmp.gzkji5A1ps + return 0 + read -r pns name + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + [[ -z restore-to-encrypted-cluster-7078 ]] + [[ -z on-demand-backup-pvc ]] + kubectl_bin patch pxc-backup -n restore-to-encrypted-cluster-7078 on-demand-backup-pvc --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.XDttRD4yJ8 ++ mktemp + local LAST_ERR=/tmp/tmp.pFjG2KN9Bu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc-backup -n restore-to-encrypted-cluster-7078 on-demand-backup-pvc --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XDttRD4yJ8 perconaxtradbclusterbackup.pxc.percona.com/on-demand-backup-pvc patched (no change) + cat /tmp/tmp.pFjG2KN9Bu + rm /tmp/tmp.XDttRD4yJ8 /tmp/tmp.pFjG2KN9Bu + return 0 + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found perconaxtradbcluster.pxc.percona.com "some-name" deleted from restore-to-encrypted-cluster-7078 namespace + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found perconaxtradbclusterbackup.pxc.percona.com "on-demand-backup-pvc" deleted from restore-to-encrypted-cluster-7078 namespace + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found perconaxtradbclusterrestore.pxc.percona.com "on-demand-backup-pvc" deleted from restore-to-encrypted-cluster-7078 namespace + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ awk '{print $1}' ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + awk '{print$1}' + xargs kubectl delete ns + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.ugHe6PYJ84 ++ mktemp + local LAST_OUT=/tmp/tmp.x5tUiaLMaK ++ mktemp + local LAST_ERR=/tmp/tmp.6GKR4F5eRm + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.hYmgvZNLuy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.x5tUiaLMaK + cat /tmp/tmp.hYmgvZNLuy + rm /tmp/tmp.x5tUiaLMaK /tmp/tmp.hYmgvZNLuy + return 0 namespace "restore-to-encrypted-cluster-7078" deleted namespace "vault-service-1-5844" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ugHe6PYJ84 namespace "pxc-operator" deleted + cat /tmp/tmp.6GKR4F5eRm + rm /tmp/tmp.ugHe6PYJ84 /tmp/tmp.6GKR4F5eRm + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.4V9RuMVMpy ++ mktemp + local LAST_ERR=/tmp/tmp.7y3HfcbXu8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4V9RuMVMpy namespace/pxc-operator created + cat /tmp/tmp.7y3HfcbXu8 + rm /tmp/tmp.4V9RuMVMpy /tmp/tmp.7y3HfcbXu8 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.DsOo6TVCjq +++ mktemp ++ local LAST_ERR=/tmp/tmp.o8ikcp9Cce ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DsOo6TVCjq ++ cat /tmp/tmp.o8ikcp9Cce ++ rm /tmp/tmp.DsOo6TVCjq /tmp/tmp.o8ikcp9Cce ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster5 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.9gxaZs1aAc ++ mktemp + local LAST_ERR=/tmp/tmp.Fw52Klld7D + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster5 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9gxaZs1aAc Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster5" modified. + cat /tmp/tmp.Fw52Klld7D + rm /tmp/tmp.9gxaZs1aAc /tmp/tmp.Fw52Klld7D + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.xHoOHv9QUG ++ mktemp + local LAST_ERR=/tmp/tmp.kTfh8JjUrz + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xHoOHv9QUG customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.kTfh8JjUrz + rm /tmp/tmp.xHoOHv9QUG /tmp/tmp.kTfh8JjUrz + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.r6C1EB1HJG ++ mktemp + local LAST_ERR=/tmp/tmp.VLtl6KUctM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.r6C1EB1HJG customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.VLtl6KUctM E0702 03:45:23.190185 12310 reflector.go:227] "Failed to watch" err="Get \"https://34.29.78.73/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusters.pxc.percona.com&resourceVersion=1782963919486132000&timeoutSeconds=318&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.r6C1EB1HJG /tmp/tmp.VLtl6KUctM + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.iUOwjAKsJo ++ mktemp + local LAST_ERR=/tmp/tmp.rRf62sK6SR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iUOwjAKsJo customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.rRf62sK6SR E0702 03:45:24.246932 12826 reflector.go:227] "Failed to watch" err="Get \"https://34.29.78.73/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterbackups.pxc.percona.com&resourceVersion=1782963919486132000&timeoutSeconds=418&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.iUOwjAKsJo /tmp/tmp.rRf62sK6SR + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.sroUulXLMo ++ mktemp + local LAST_ERR=/tmp/tmp.oQkT50Ekwk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sroUulXLMo customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.oQkT50Ekwk E0702 03:45:26.516330 12939 reflector.go:227] "Failed to watch" err="Get \"https://34.29.78.73/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterrestores.pxc.percona.com&resourceVersion=1782963924486188000&timeoutSeconds=339&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.sroUulXLMo /tmp/tmp.oQkT50Ekwk + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.PQVHECzYv0 ++ mktemp + local LAST_ERR=/tmp/tmp.yvyTciVxwL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PQVHECzYv0 clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.yvyTciVxwL + rm /tmp/tmp.PQVHECzYv0 /tmp/tmp.yvyTciVxwL + return 0 + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-operator.yaml + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a^' + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - ++ mktemp + local LAST_OUT=/tmp/tmp.dSakPE2SvK ++ mktemp + local LAST_ERR=/tmp/tmp.EKVjalYz57 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dSakPE2SvK deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.EKVjalYz57 + rm /tmp/tmp.dSakPE2SvK /tmp/tmp.EKVjalYz57 + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.7twyTHLRDV ++ mktemp + local LAST_ERR=/tmp/tmp.AbJB3gZaX9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7twyTHLRDV pod/percona-xtradb-cluster-operator-65b5d85575-xmkvj condition met + cat /tmp/tmp.AbJB3gZaX9 E0702 03:45:48.573687 15872 reflector.go:227] "Failed to watch" err="Get \"https://34.29.78.73/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-xmkvj&resourceVersion=1782963948128526000&timeoutSeconds=587&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.7twyTHLRDV /tmp/tmp.AbJB3gZaX9 + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Wchhb2MVzX +++ mktemp ++ local LAST_ERR=/tmp/tmp.lChVXxJini ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Wchhb2MVzX ++ cat /tmp/tmp.lChVXxJini ++ rm /tmp/tmp.Wchhb2MVzX /tmp/tmp.lChVXxJini ++ return 0 + wait_pod percona-xtradb-cluster-operator-65b5d85575-xmkvj 480 pxc-operator + local pod=percona-xtradb-cluster-operator-65b5d85575-xmkvj + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-65b5d85575-xmkvj ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-65b5d85575-xmkvj condition met E0702 03:45:54.048947 17035 reflector.go:227] "Failed to watch" err="Get \"https://34.29.78.73/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-xmkvj&resourceVersion=1782963953067190000&timeoutSeconds=507&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-65b5d85575-xmkvj to become Ready.Ok + sleep 3 + create_namespace restore-to-encrypted-cluster-27994 + local namespace=restore-to-encrypted-cluster-27994 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces restore-to-encrypted-cluster-27994' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces restore-to-encrypted-cluster-27994 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace restore-to-encrypted-cluster-27994 ++ mktemp + local LAST_OUT=/tmp/tmp.yhpe4AwLHJ + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + xargs kubectl delete ns + awk '{print$1}' ++ mktemp + kubectl_bin get ns ++ mktemp + local LAST_ERR=/tmp/tmp.nNgRWqLjj7 + local exit_status=0 + local LAST_OUT=/tmp/tmp.DiHWgwKq4s ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace restore-to-encrypted-cluster-27994 ++ mktemp + local LAST_ERR=/tmp/tmp.Fy4mj8e5hK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace restore-to-encrypted-cluster-27994 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DiHWgwKq4s + cat /tmp/tmp.Fy4mj8e5hK + rm /tmp/tmp.DiHWgwKq4s /tmp/tmp.Fy4mj8e5hK + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace restore-to-encrypted-cluster-27994 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.yhpe4AwLHJ + cat /tmp/tmp.nNgRWqLjj7 Error from server (NotFound): namespaces "restore-to-encrypted-cluster-27994" not found + rm /tmp/tmp.yhpe4AwLHJ /tmp/tmp.nNgRWqLjj7 + return 1 + : + wait_for_delete namespace/restore-to-encrypted-cluster-27994 + local res=namespace/restore-to-encrypted-cluster-27994 + echo -n 'waiting for namespace/restore-to-encrypted-cluster-27994 to be deleted' waiting for namespace/restore-to-encrypted-cluster-27994 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "restore-to-encrypted-cluster-27994" not found + desc 'create namespace restore-to-encrypted-cluster-27994' + set +o xtrace ----------------------------------------------------------------------------------- create namespace restore-to-encrypted-cluster-27994 ----------------------------------------------------------------------------------- + kubectl_bin create namespace restore-to-encrypted-cluster-27994 ++ mktemp + local LAST_OUT=/tmp/tmp.9rNy3NQvVq ++ mktemp + local LAST_ERR=/tmp/tmp.iqhxpvVSLE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace restore-to-encrypted-cluster-27994 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9rNy3NQvVq namespace/restore-to-encrypted-cluster-27994 created + cat /tmp/tmp.iqhxpvVSLE + rm /tmp/tmp.9rNy3NQvVq /tmp/tmp.iqhxpvVSLE + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.F8pDQ6AsPB +++ mktemp ++ local LAST_ERR=/tmp/tmp.mErHn1ZwSH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.F8pDQ6AsPB ++ cat /tmp/tmp.mErHn1ZwSH ++ rm /tmp/tmp.F8pDQ6AsPB /tmp/tmp.mErHn1ZwSH ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster5 --namespace=restore-to-encrypted-cluster-27994 ++ mktemp + local LAST_OUT=/tmp/tmp.4dCagWsjSb ++ mktemp + local LAST_ERR=/tmp/tmp.TIZQvKPv8P + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster5 --namespace=restore-to-encrypted-cluster-27994 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4dCagWsjSb Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster5" modified. + cat /tmp/tmp.TIZQvKPv8P + rm /tmp/tmp.4dCagWsjSb /tmp/tmp.TIZQvKPv8P + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.C9r5LnMlq0 ++ mktemp + local LAST_ERR=/tmp/tmp.1mag4eecl8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.C9r5LnMlq0 secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.1mag4eecl8 + rm /tmp/tmp.C9r5LnMlq0 /tmp/tmp.1mag4eecl8 + return 0 + cluster=some-name + spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.JOFZlN0B86 ++ mktemp + local LAST_ERR=/tmp/tmp.mJIOLYCD9M + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JOFZlN0B86 secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.mJIOLYCD9M + rm /tmp/tmp.JOFZlN0B86 /tmp/tmp.mJIOLYCD9M + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.restore-to-encrypted-cluster-27994~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + local LAST_OUT=/tmp/tmp.rjSELbXsVP ++ mktemp + local LAST_ERR=/tmp/tmp.P9NKeWG2pK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rjSELbXsVP deployment.apps/pxc-client created + cat /tmp/tmp.P9NKeWG2pK + rm /tmp/tmp.rjSELbXsVP /tmp/tmp.P9NKeWG2pK + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml + local pvc_name= + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#' ++ mktemp + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/some-name.yml + local LAST_OUT=/tmp/tmp.mCrt9WfhWU + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.restore-to-encrypted-cluster-27994~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + local LAST_ERR=/tmp/tmp.iV5n4OBLZ8 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mCrt9WfhWU perconaxtradbcluster.pxc.percona.com/some-name created + cat /tmp/tmp.iV5n4OBLZ8 + rm /tmp/tmp.mCrt9WfhWU /tmp/tmp.iV5n4OBLZ8 + return 0 + desc 'waiting pods to be running for some-name' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for some-name ----------------------------------------------------------------------------------- ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Xx1UdJkxdp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yYdMCEpoNM +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Xx1UdJkxdp +++ cat /tmp/tmp.yYdMCEpoNM +++ rm /tmp/tmp.Xx1UdJkxdp /tmp/tmp.yYdMCEpoNM +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3CjdhOvA4f ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LfO79kUbAK +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.3CjdhOvA4f +++ cat /tmp/tmp.LfO79kUbAK +++ rm /tmp/tmp.3CjdhOvA4f /tmp/tmp.LfO79kUbAK +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo some-name-proxysql ++ return + local proxy=some-name-proxysql ++ get_proxy_size some-name ++ local cluster=some-name ++ local size=0 +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NwU2f3ZobM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7UIrUCit51 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.NwU2f3ZobM +++ cat /tmp/tmp.7UIrUCit51 +++ rm /tmp/tmp.NwU2f3ZobM /tmp/tmp.7UIrUCit51 +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jHZbj0kiqa ++++ mktemp +++ local LAST_ERR=/tmp/tmp.69D2qtUWJ4 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.jHZbj0kiqa +++ cat /tmp/tmp.69D2qtUWJ4 +++ rm /tmp/tmp.jHZbj0kiqa /tmp/tmp.69D2qtUWJ4 +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9vjKplJliR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3XGnztdVXB +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.9vjKplJliR +++ cat /tmp/tmp.3XGnztdVXB +++ rm /tmp/tmp.9vjKplJliR /tmp/tmp.3XGnztdVXB +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running some-name-pxc 3 + local name=some-name-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z some-name-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for some-name-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for some-name-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-0 480 + local pod=some-name-pxc-0 + local max_retry=480 + local ns= ++ echo some-name-pxc-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/some-name-pxc-0 condition met waiting for pod/some-name-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-1 480 + local pod=some-name-pxc-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo some-name-pxc-1 + local container=pxc + set +o xtrace pod/some-name-pxc-1 condition met waiting for pod/some-name-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-pxc-2 480 + local pod=some-name-pxc-2 + local max_retry=480 + local ns= ++ echo some-name-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/some-name-pxc-2 condition met waiting for pod/some-name-pxc-2 to become Ready.Ok + wait_for_running some-name-proxysql 2 + local name=some-name-proxysql + local size=2 + local last_pod + local max_retry=480 + [[ -z some-name-proxysql ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for some-name-proxysql' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for some-name-proxysql ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in '$(seq 0 $last_pod)' + wait_pod some-name-proxysql-0 480 + local pod=some-name-proxysql-0 + local max_retry=480 + local ns= ++ echo some-name-proxysql-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace pod/some-name-proxysql-0 condition met E0702 03:50:59.353164 29286 reflector.go:227] "Failed to watch" err="Get \"https://34.29.78.73/api/v1/namespaces/restore-to-encrypted-cluster-27994/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-proxysql-0&resourceVersion=1782964259155045000&timeoutSeconds=359&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/some-name-proxysql-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod some-name-proxysql-1 480 + local pod=some-name-proxysql-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo some-name-proxysql-1 + local container=proxysql + set +o xtrace pod/some-name-proxysql-1 condition met E0702 03:51:09.656939 30915 reflector.go:227] "Failed to watch" err="Get \"https://34.29.78.73/api/v1/namespaces/restore-to-encrypted-cluster-27994/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-proxysql-1&resourceVersion=1782964269155108000&timeoutSeconds=579&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/some-name-proxysql-1 to become Ready.Ok + wait_cluster_consistency some-name 3 2 + local cluster_name=some-name + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/some-name to be ready' waiting for pxc/some-name to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oRlS2gWNwI +++ mktemp ++ local LAST_ERR=/tmp/tmp.SuFQ7jFSxb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oRlS2gWNwI ++ cat /tmp/tmp.SuFQ7jFSxb ++ rm /tmp/tmp.oRlS2gWNwI /tmp/tmp.SuFQ7jFSxb ++ return 0 + state=ready ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.P1U2dTepRh +++ mktemp ++ local LAST_ERR=/tmp/tmp.dcf9V0nXdl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.P1U2dTepRh ++ cat /tmp/tmp.dcf9V0nXdl ++ rm /tmp/tmp.P1U2dTepRh /tmp/tmp.dcf9V0nXdl ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine some-name +++ local cluster_name=some-name ++++ get_proxy some-name ++++ local target_cluster=some-name +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.LPZgZOyuG6 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.ydacTPx2bF +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.LPZgZOyuG6 +++++ cat /tmp/tmp.ydacTPx2bF +++++ rm /tmp/tmp.LPZgZOyuG6 /tmp/tmp.ydacTPx2bF +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.NGAmsA0Oi0 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.KQhnmBwWbx +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.NGAmsA0Oi0 +++++ cat /tmp/tmp.KQhnmBwWbx +++++ rm /tmp/tmp.NGAmsA0Oi0 /tmp/tmp.KQhnmBwWbx +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo some-name-proxysql ++++ return +++ local cluster_proxy=some-name-proxysql +++ echo proxysql ++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gylZJbsFYY +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ia2DN4tijA ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.gylZJbsFYY ++ cat /tmp/tmp.Ia2DN4tijA ++ rm /tmp/tmp.gylZJbsFYY /tmp/tmp.Ia2DN4tijA ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.aUzokJPisz +++ mktemp ++ local LAST_ERR=/tmp/tmp.q8626TzEzS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.aUzokJPisz ++ cat /tmp/tmp.q8626TzEzS ++ rm /tmp/tmp.aUzokJPisz /tmp/tmp.q8626TzEzS ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rm7gLMDArK +++ mktemp ++ local LAST_ERR=/tmp/tmp.lW8aVqJQ1Z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rm7gLMDArK ++ cat /tmp/tmp.lW8aVqJQ1Z ++ rm /tmp/tmp.rm7gLMDArK /tmp/tmp.lW8aVqJQ1Z ++ return 0 + client_pod=pxc-client-67fc4995bb-8kk9c + wait_pod pxc-client-67fc4995bb-8kk9c + local pod=pxc-client-67fc4995bb-8kk9c + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-8kk9c + local container= + set +o xtrace pod/pxc-client-67fc4995bb-8kk9c condition met E0702 03:51:50.574078 4464 reflector.go:227] "Failed to watch" err="Get \"https://34.29.78.73/api/v1/namespaces/restore-to-encrypted-cluster-27994/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-8kk9c&resourceVersion=1782964307631567019&timeoutSeconds=332&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-8kk9c to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace error: cannot exec into a container in a completed pod; current phase is Failed + write_data some-name 100500 no-proxy -P3306 + local cluster=some-name + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password some-name root ++ local cluster=some-name ++ local user=root +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.t85En9h0jd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4wTSMun4CS +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.t85En9h0jd +++ cat /tmp/tmp.4wTSMun4CS +++ rm /tmp/tmp.t85En9h0jd /tmp/tmp.4wTSMun4CS +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2RZRYjY8w5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.BLjQZAs7N2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2RZRYjY8w5 ++ cat /tmp/tmp.BLjQZAs7N2 ++ rm /tmp/tmp.2RZRYjY8w5 /tmp/tmp.BLjQZAs7N2 ++ return 0 + root_pass=root_password ++ get_proxy some-name ++ local target_cluster=some-name +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.SHUI2Iavej ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8BkBEvquCV +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.SHUI2Iavej +++ cat /tmp/tmp.8BkBEvquCV +++ rm /tmp/tmp.SHUI2Iavej /tmp/tmp.8BkBEvquCV +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yCb6o4hcMO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TdR4nkt30k +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.yCb6o4hcMO +++ cat /tmp/tmp.TdR4nkt30k +++ rm /tmp/tmp.yCb6o4hcMO /tmp/tmp.TdR4nkt30k +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo some-name-proxysql ++ return + proxy=some-name-proxysql + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h some-name-pxc-0.some-name-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h some-name-pxc-0.some-name-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V7pw9PENsd +++ mktemp ++ local LAST_ERR=/tmp/tmp.dhNkjMo4dv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.V7pw9PENsd ++ cat /tmp/tmp.dhNkjMo4dv ++ rm /tmp/tmp.V7pw9PENsd /tmp/tmp.dhNkjMo4dv ++ return 0 + client_pod=pxc-client-67fc4995bb-8kk9c + wait_pod pxc-client-67fc4995bb-8kk9c + local pod=pxc-client-67fc4995bb-8kk9c + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-8kk9c + local container= + set +o xtrace Error from server (NotFound): pods "pxc-client-67fc4995bb-8kk9c" not found waiting for pod/pxc-client-67fc4995bb-8kk9c to become Ready................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Error from server (NotFound): pods "pxc-client-67fc4995bb-8kk9c" not found