Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/logs/backup-storage-tls-8-0.log Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 + GTID_PATTERN='[A-F0-9a-f]{8}-[A-F0-9a-f]{4}-[A-F0-9a-f]{4}-[A-F0-9a-f]{4}-[A-F0-9a-f]{12}:[0-9]+' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + main + create_infra backup-storage-tls-14309 + local ns=backup-storage-tls-14309 + '[' -n pxc-operator ']' + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrole ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + kubectl_bin get ns + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.3ZP70bIVb0 ++ mktemp + local LAST_ERR=/tmp/tmp.zsB2455aGY + local exit_status=0 ++ mktemp + local LAST_OUT=/tmp/tmp.b4KmR8oEeO ++ seq 0 2 ++ mktemp + local LAST_ERR=/tmp/tmp.EKgCidqwwB + local exit_status=0 + for i in $(seq 0 2) + set +e + kubectl delete namespace pxc-operator ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.b4KmR8oEeO + cat /tmp/tmp.EKgCidqwwB + rm /tmp/tmp.b4KmR8oEeO /tmp/tmp.EKgCidqwwB + return 0 namespace "backup-storage-tls-7678" deleted namespace "cert-manager" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3ZP70bIVb0 namespace "pxc-operator" deleted + cat /tmp/tmp.zsB2455aGY + rm /tmp/tmp.3ZP70bIVb0 /tmp/tmp.zsB2455aGY + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.bAups95BD8 ++ mktemp + local LAST_ERR=/tmp/tmp.3JxDa9rP05 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bAups95BD8 namespace/pxc-operator created + cat /tmp/tmp.3JxDa9rP05 + rm /tmp/tmp.bAups95BD8 /tmp/tmp.3JxDa9rP05 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.in6YvrX6vB +++ mktemp ++ local LAST_ERR=/tmp/tmp.uTDWsLAS8H ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.in6YvrX6vB ++ cat /tmp/tmp.uTDWsLAS8H ++ rm /tmp/tmp.in6YvrX6vB /tmp/tmp.uTDWsLAS8H ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster4 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ax3L8bUVsp ++ mktemp + local LAST_ERR=/tmp/tmp.Ux5RgvNHDZ + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster4 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ax3L8bUVsp Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster4" modified. + cat /tmp/tmp.Ux5RgvNHDZ + rm /tmp/tmp.ax3L8bUVsp /tmp/tmp.Ux5RgvNHDZ + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.YbKcn6SDQS ++ mktemp + local LAST_ERR=/tmp/tmp.CRNFqYyP2m + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YbKcn6SDQS customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.CRNFqYyP2m + rm /tmp/tmp.YbKcn6SDQS /tmp/tmp.CRNFqYyP2m + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.XdoVAGY3eD ++ mktemp + local LAST_ERR=/tmp/tmp.k3pCqNv3sW + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.XdoVAGY3eD customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.k3pCqNv3sW + rm /tmp/tmp.XdoVAGY3eD /tmp/tmp.k3pCqNv3sW + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.rByku2MM1k ++ mktemp + local LAST_ERR=/tmp/tmp.qJXiw5m1Kd + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rByku2MM1k customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.qJXiw5m1Kd + rm /tmp/tmp.rByku2MM1k /tmp/tmp.qJXiw5m1Kd + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.QSCxvRWpLM ++ mktemp + local LAST_ERR=/tmp/tmp.McAbBQ3PXE + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QSCxvRWpLM customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.McAbBQ3PXE + rm /tmp/tmp.QSCxvRWpLM /tmp/tmp.McAbBQ3PXE + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-rbac.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.MWfueDIizz ++ mktemp + local LAST_ERR=/tmp/tmp.UKsje8vLj1 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MWfueDIizz clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.UKsje8vLj1 + rm /tmp/tmp.MWfueDIizz /tmp/tmp.UKsje8vLj1 + return 0 + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-operator.yaml + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420^' + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.PWZ7jPxNim ++ mktemp + local LAST_ERR=/tmp/tmp.qb25dejrya + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PWZ7jPxNim deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.qb25dejrya + rm /tmp/tmp.PWZ7jPxNim /tmp/tmp.qb25dejrya + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.9Ca9AgD70z ++ mktemp + local LAST_ERR=/tmp/tmp.4ZmTHCzHdc + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9Ca9AgD70z pod/percona-xtradb-cluster-operator-658fcf4b98-zwqb7 condition met + cat /tmp/tmp.4ZmTHCzHdc + rm /tmp/tmp.9Ca9AgD70z /tmp/tmp.4ZmTHCzHdc + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.uh0ocSF6H6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.LTT4n0y3BK ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uh0ocSF6H6 ++ cat /tmp/tmp.LTT4n0y3BK ++ rm /tmp/tmp.uh0ocSF6H6 /tmp/tmp.LTT4n0y3BK ++ return 0 + wait_pod percona-xtradb-cluster-operator-658fcf4b98-zwqb7 480 pxc-operator + local pod=percona-xtradb-cluster-operator-658fcf4b98-zwqb7 + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-658fcf4b98-zwqb7 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-658fcf4b98-zwqb7 condition met waiting for pod/percona-xtradb-cluster-operator-658fcf4b98-zwqb7 to become Ready.Ok + sleep 3 + create_namespace backup-storage-tls-14309 + local namespace=backup-storage-tls-14309 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrole ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces backup-storage-tls-14309' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces backup-storage-tls-14309 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace backup-storage-tls-14309 + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.JZ0wYYSbBo ++ mktemp + local LAST_OUT=/tmp/tmp.sTgQHHnntT ++ mktemp + local LAST_ERR=/tmp/tmp.L8SPAY5iwa + local exit_status=0 ++ mktemp ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace backup-storage-tls-14309 + local LAST_ERR=/tmp/tmp.gEqeO2BmjK + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace backup-storage-tls-14309 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.sTgQHHnntT + cat /tmp/tmp.gEqeO2BmjK + rm /tmp/tmp.sTgQHHnntT /tmp/tmp.gEqeO2BmjK + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace backup-storage-tls-14309 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.JZ0wYYSbBo + cat /tmp/tmp.L8SPAY5iwa Error from server (NotFound): namespaces "backup-storage-tls-14309" not found + rm /tmp/tmp.JZ0wYYSbBo /tmp/tmp.L8SPAY5iwa + return 1 + : + wait_for_delete namespace/backup-storage-tls-14309 + local res=namespace/backup-storage-tls-14309 + echo -n 'waiting for namespace/backup-storage-tls-14309 to be deleted' waiting for namespace/backup-storage-tls-14309 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "backup-storage-tls-14309" not found + desc 'create namespace backup-storage-tls-14309' + set +o xtrace ----------------------------------------------------------------------------------- create namespace backup-storage-tls-14309 ----------------------------------------------------------------------------------- + kubectl_bin create namespace backup-storage-tls-14309 ++ mktemp + local LAST_OUT=/tmp/tmp.ogrB3kgzV9 ++ mktemp + local LAST_ERR=/tmp/tmp.36nudiwEo3 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace backup-storage-tls-14309 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ogrB3kgzV9 namespace/backup-storage-tls-14309 created + cat /tmp/tmp.36nudiwEo3 + rm /tmp/tmp.ogrB3kgzV9 /tmp/tmp.36nudiwEo3 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.ripluC5rnD +++ mktemp ++ local LAST_ERR=/tmp/tmp.Fv8vkfp48W ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ripluC5rnD ++ cat /tmp/tmp.Fv8vkfp48W ++ rm /tmp/tmp.ripluC5rnD /tmp/tmp.Fv8vkfp48W ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster4 --namespace=backup-storage-tls-14309 ++ mktemp + local LAST_OUT=/tmp/tmp.HWNtK4XS8H ++ mktemp + local LAST_ERR=/tmp/tmp.CxfrLHwAVh + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster4 --namespace=backup-storage-tls-14309 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HWNtK4XS8H Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster4" modified. + cat /tmp/tmp.CxfrLHwAVh + rm /tmp/tmp.HWNtK4XS8H /tmp/tmp.CxfrLHwAVh + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.93yPrYGH4P ++ mktemp + local LAST_ERR=/tmp/tmp.VUQ1vogDlE + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.93yPrYGH4P secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.VUQ1vogDlE + rm /tmp/tmp.93yPrYGH4P /tmp/tmp.VUQ1vogDlE + return 0 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.z4qV1tJm1R ++ mktemp + local LAST_ERR=/tmp/tmp.I51vUgJ9TI + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.z4qV1tJm1R namespace/cert-manager created + cat /tmp/tmp.I51vUgJ9TI + rm /tmp/tmp.z4qV1tJm1R /tmp/tmp.I51vUgJ9TI + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.KGVPYB2Cvn ++ mktemp + local LAST_ERR=/tmp/tmp.VvK2H90JQV + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.KGVPYB2Cvn namespace/cert-manager labeled + cat /tmp/tmp.VvK2H90JQV + rm /tmp/tmp.KGVPYB2Cvn /tmp/tmp.VvK2H90JQV + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.8AMxo1MCEa ++ mktemp + local LAST_ERR=/tmp/tmp.TKFitHfqsz + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.8AMxo1MCEa namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.TKFitHfqsz Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.8AMxo1MCEa /tmp/tmp.TKFitHfqsz + return 0 + '[' '' == 4.10 ']' + kubectl_bin -n cert-manager rollout status deployment/cert-manager --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.2ur4nsu2s1 ++ mktemp + local LAST_ERR=/tmp/tmp.zGogLcHc7b + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager rollout status deployment/cert-manager --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2ur4nsu2s1 deployment "cert-manager" successfully rolled out + cat /tmp/tmp.zGogLcHc7b + rm /tmp/tmp.2ur4nsu2s1 /tmp/tmp.zGogLcHc7b + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.AXKJO7M2c0 ++ mktemp + local LAST_ERR=/tmp/tmp.emmdyEDrbU + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AXKJO7M2c0 deployment "cert-manager-cainjector" successfully rolled out + cat /tmp/tmp.emmdyEDrbU + rm /tmp/tmp.AXKJO7M2c0 /tmp/tmp.emmdyEDrbU + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.RPCRu6Qtgl ++ mktemp + local LAST_ERR=/tmp/tmp.LD9OpRoVw6 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RPCRu6Qtgl Waiting for deployment "cert-manager-webhook" rollout to finish: 0 of 1 updated replicas are available... deployment "cert-manager-webhook" successfully rolled out + cat /tmp/tmp.LD9OpRoVw6 + rm /tmp/tmp.RPCRu6Qtgl /tmp/tmp.LD9OpRoVw6 + return 0 + log 'waiting for cert-manager webhook caBundle to be injected' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T12:15:39+0000]' waiting for cert-manager webhook caBundle to be injected [2026-10-05T12:15:39+0000] waiting for cert-manager webhook caBundle to be injected ++ date +%s + local deadline=1791202659 + grep -q . + kubectl_bin get validatingwebhookconfigurations cert-manager-webhook -o 'jsonpath={.webhooks[0].clientConfig.caBundle}' + log 'probing cert-manager webhook' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T12:15:41+0000]' probing cert-manager webhook [2026-10-05T12:15:41+0000] probing cert-manager webhook + local probe_file ++ mktemp + probe_file=/tmp/tmp.iCGlG2Flda + cat + local retries=0 + kubectl_bin apply -f /tmp/tmp.iCGlG2Flda --dry-run=server + rm -f /tmp/tmp.iCGlG2Flda + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/backup-storage-tls/conf/issuer.yml ++ mktemp + local LAST_OUT=/tmp/tmp.bohOj1QVSu ++ mktemp + local LAST_ERR=/tmp/tmp.YnqN0Qw4MB + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/backup-storage-tls/conf/issuer.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bohOj1QVSu issuer.cert-manager.io/selfsigning-issuer created + cat /tmp/tmp.YnqN0Qw4MB + rm /tmp/tmp.bohOj1QVSu /tmp/tmp.YnqN0Qw4MB + return 0 + kubectl_bin apply -f - + sed s/#namespace/backup-storage-tls-14309/g /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/backup-storage-tls/conf/cert.yml ++ mktemp + local LAST_OUT=/tmp/tmp.jAnzdcX6SQ ++ mktemp + local LAST_ERR=/tmp/tmp.lfgj32xokl + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jAnzdcX6SQ certificate.cert-manager.io/minio-tls-certs created + cat /tmp/tmp.lfgj32xokl + rm /tmp/tmp.jAnzdcX6SQ /tmp/tmp.lfgj32xokl + return 0 + sleep 30 + minio_cert_secret=minio-tls-certs + start_minio minio-tls-certs + deploy_helm backup-storage-tls-14309 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "hashicorp" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret=minio-tls-certs + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G') + local minio_args + [[ -n minio-tls-certs ]] + endpoint=https://minio-service:9000 + minio_args+=(--set tls.enabled=true --set tls.certSecret="$cert_secret" --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key) + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: failed post-install: 1 error occurred: * timed out waiting for the condition + [[ 1 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 2 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 3 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 4 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 5 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 6 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 7 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 8 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 9 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 10 -ge 10 ]] + echo 'The command '\''helm' install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key 'minio/minio'\'' has failed after 10 attempts.' The command 'helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set users[0].accessKey=some-access-key --set users[0].secretKey=some-secret-key --set users[0].policy=consoleAdmin --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio' has failed after 10 attempts. + exit 1