Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/logs/pitr-8-0.log Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + main + create_infra pitr-2340 + local ns=pitr-2340 + '[' -n pxc-operator ']' + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrole ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns + awk '{print$1}' ++ mktemp + local LAST_OUT=/tmp/tmp.5Yd19DDrWg + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.FQM8PkvY9g ++ mktemp + local LAST_ERR=/tmp/tmp.x3IXVr3XaE + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.E2fDvedhb9 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5Yd19DDrWg + cat /tmp/tmp.x3IXVr3XaE + rm /tmp/tmp.5Yd19DDrWg /tmp/tmp.x3IXVr3XaE + return 0 namespace "cert-manager" deleted namespace "pitr-15711" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FQM8PkvY9g namespace "pxc-operator" deleted + cat /tmp/tmp.E2fDvedhb9 + rm /tmp/tmp.FQM8PkvY9g /tmp/tmp.E2fDvedhb9 + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.v1Ws54vVbV ++ mktemp + local LAST_ERR=/tmp/tmp.Chgc1YSoYr + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.v1Ws54vVbV namespace/pxc-operator created + cat /tmp/tmp.Chgc1YSoYr + rm /tmp/tmp.v1Ws54vVbV /tmp/tmp.Chgc1YSoYr + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.DDlgxKYrom +++ mktemp ++ local LAST_ERR=/tmp/tmp.6mvFfFnigW ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DDlgxKYrom ++ cat /tmp/tmp.6mvFfFnigW ++ rm /tmp/tmp.DDlgxKYrom /tmp/tmp.6mvFfFnigW ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster14 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.mi6YvdkAZG ++ mktemp + local LAST_ERR=/tmp/tmp.sjxtewHPr1 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster14 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mi6YvdkAZG Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster14" modified. + cat /tmp/tmp.sjxtewHPr1 + rm /tmp/tmp.mi6YvdkAZG /tmp/tmp.sjxtewHPr1 + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.Om8Hyq3XKk ++ mktemp + local LAST_ERR=/tmp/tmp.SEROfSS7Mp + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Om8Hyq3XKk customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.SEROfSS7Mp + rm /tmp/tmp.Om8Hyq3XKk /tmp/tmp.SEROfSS7Mp + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.gfYYxVgpSt ++ mktemp + local LAST_ERR=/tmp/tmp.JcSUeC5hRm + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gfYYxVgpSt customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.JcSUeC5hRm + rm /tmp/tmp.gfYYxVgpSt /tmp/tmp.JcSUeC5hRm + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.cyUzEMrMB6 ++ mktemp + local LAST_ERR=/tmp/tmp.NggIsrTTsE + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cyUzEMrMB6 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.NggIsrTTsE + rm /tmp/tmp.cyUzEMrMB6 /tmp/tmp.NggIsrTTsE + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.a8DjyLAiDh ++ mktemp + local LAST_ERR=/tmp/tmp.j2SsQnj36Y + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.a8DjyLAiDh customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.j2SsQnj36Y + rm /tmp/tmp.a8DjyLAiDh /tmp/tmp.j2SsQnj36Y + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-rbac.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.dLQXAGOz76 ++ mktemp + local LAST_ERR=/tmp/tmp.wXQ9ASyIsg + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.dLQXAGOz76 clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.wXQ9ASyIsg + rm /tmp/tmp.dLQXAGOz76 /tmp/tmp.wXQ9ASyIsg + return 0 + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - ++ mktemp + local LAST_OUT=/tmp/tmp.fIebtOjO04 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' ++ mktemp + local LAST_ERR=/tmp/tmp.rJzphol5zt + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fIebtOjO04 deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.rJzphol5zt + rm /tmp/tmp.fIebtOjO04 /tmp/tmp.rJzphol5zt + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.osByr4yIOP ++ mktemp + local LAST_ERR=/tmp/tmp.YyNmS7Xfmx + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.osByr4yIOP pod/percona-xtradb-cluster-operator-658fcf4b98-rcmqk condition met + cat /tmp/tmp.YyNmS7Xfmx + rm /tmp/tmp.osByr4yIOP /tmp/tmp.YyNmS7Xfmx + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ local LAST_OUT=/tmp/tmp.up1jV8Anhv +++ mktemp ++ local LAST_ERR=/tmp/tmp.2sT3TMZqXR ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.up1jV8Anhv ++ cat /tmp/tmp.2sT3TMZqXR ++ rm /tmp/tmp.up1jV8Anhv /tmp/tmp.2sT3TMZqXR ++ return 0 + wait_pod percona-xtradb-cluster-operator-658fcf4b98-rcmqk 480 pxc-operator + local pod=percona-xtradb-cluster-operator-658fcf4b98-rcmqk + local max_retry=480 + local ns=pxc-operator ++ grep -E '^(pxc|proxysql)$' ++ echo percona-xtradb-cluster-operator-658fcf4b98-rcmqk ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-658fcf4b98-rcmqk condition met waiting for pod/percona-xtradb-cluster-operator-658fcf4b98-rcmqk to become Ready.Ok + sleep 3 + create_namespace pitr-2340 + local namespace=pitr-2340 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pitr-2340' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pitr-2340 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pitr-2340 ++ mktemp + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + xargs kubectl delete ns + kubectl_bin get ns + local LAST_OUT=/tmp/tmp.kYV0BJZdAC ++ mktemp + local LAST_OUT=/tmp/tmp.NAENrxtVLd ++ mktemp + local LAST_ERR=/tmp/tmp.611amWmvMz + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.bJoIJRykvQ + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace pitr-2340 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NAENrxtVLd + cat /tmp/tmp.bJoIJRykvQ + rm /tmp/tmp.NAENrxtVLd /tmp/tmp.bJoIJRykvQ + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace pitr-2340 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace pitr-2340 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.kYV0BJZdAC + cat /tmp/tmp.611amWmvMz Error from server (NotFound): namespaces "pitr-2340" not found + rm /tmp/tmp.kYV0BJZdAC /tmp/tmp.611amWmvMz + return 1 + : + wait_for_delete namespace/pitr-2340 + local res=namespace/pitr-2340 + echo -n 'waiting for namespace/pitr-2340 to be deleted' waiting for namespace/pitr-2340 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pitr-2340" not found + desc 'create namespace pitr-2340' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pitr-2340 ----------------------------------------------------------------------------------- + kubectl_bin create namespace pitr-2340 ++ mktemp + local LAST_OUT=/tmp/tmp.9IGfzCmgbe ++ mktemp + local LAST_ERR=/tmp/tmp.G6NunGjrB6 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pitr-2340 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9IGfzCmgbe namespace/pitr-2340 created + cat /tmp/tmp.G6NunGjrB6 + rm /tmp/tmp.9IGfzCmgbe /tmp/tmp.G6NunGjrB6 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.QdItBGWyGn +++ mktemp ++ local LAST_ERR=/tmp/tmp.PxtrTTgzdG ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QdItBGWyGn ++ cat /tmp/tmp.PxtrTTgzdG ++ rm /tmp/tmp.QdItBGWyGn /tmp/tmp.PxtrTTgzdG ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster14 --namespace=pitr-2340 ++ mktemp + local LAST_OUT=/tmp/tmp.d7BhQ5HDLW ++ mktemp + local LAST_ERR=/tmp/tmp.hIubqAgAZI + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster14 --namespace=pitr-2340 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.d7BhQ5HDLW Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster14" modified. + cat /tmp/tmp.hIubqAgAZI + rm /tmp/tmp.d7BhQ5HDLW /tmp/tmp.hIubqAgAZI + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.OIQ7yh5AA7 ++ mktemp + local LAST_ERR=/tmp/tmp.sNpOjyd6J9 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.OIQ7yh5AA7 secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.sNpOjyd6J9 + rm /tmp/tmp.OIQ7yh5AA7 /tmp/tmp.sNpOjyd6J9 + return 0 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.3JG4d6RUPW ++ mktemp + local LAST_ERR=/tmp/tmp.WisItTZd9R + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3JG4d6RUPW namespace/cert-manager created + cat /tmp/tmp.WisItTZd9R + rm /tmp/tmp.3JG4d6RUPW /tmp/tmp.WisItTZd9R + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.vY0HFbQmp4 ++ mktemp + local LAST_ERR=/tmp/tmp.hZV0GtOiEF + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vY0HFbQmp4 namespace/cert-manager labeled + cat /tmp/tmp.hZV0GtOiEF + rm /tmp/tmp.vY0HFbQmp4 /tmp/tmp.hZV0GtOiEF + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.3KWa1ARbNJ ++ mktemp + local LAST_ERR=/tmp/tmp.OgTHVeEOhi + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3KWa1ARbNJ namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.OgTHVeEOhi Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.3KWa1ARbNJ /tmp/tmp.OgTHVeEOhi + return 0 + '[' '' == 4.10 ']' + kubectl_bin -n cert-manager rollout status deployment/cert-manager --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.6BURhXYCGb ++ mktemp + local LAST_ERR=/tmp/tmp.kGRiyPlDdv + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager rollout status deployment/cert-manager --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6BURhXYCGb deployment "cert-manager" successfully rolled out + cat /tmp/tmp.kGRiyPlDdv + rm /tmp/tmp.6BURhXYCGb /tmp/tmp.kGRiyPlDdv + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.fgETEaZoaw ++ mktemp + local LAST_ERR=/tmp/tmp.rsMJ2126Xw + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fgETEaZoaw deployment "cert-manager-cainjector" successfully rolled out + cat /tmp/tmp.rsMJ2126Xw + rm /tmp/tmp.fgETEaZoaw /tmp/tmp.rsMJ2126Xw + return 0 + kubectl_bin -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.LRcm4uLj6r ++ mktemp + local LAST_ERR=/tmp/tmp.t7pYewhc6M + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LRcm4uLj6r Waiting for deployment "cert-manager-webhook" rollout to finish: 0 of 1 updated replicas are available... deployment "cert-manager-webhook" successfully rolled out + cat /tmp/tmp.t7pYewhc6M + rm /tmp/tmp.LRcm4uLj6r /tmp/tmp.t7pYewhc6M + return 0 + log 'waiting for cert-manager webhook caBundle to be injected' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T13:29:57+0000]' waiting for cert-manager webhook caBundle to be injected [2026-10-05T13:29:57+0000] waiting for cert-manager webhook caBundle to be injected ++ date +%s + local deadline=1791207117 + kubectl_bin get validatingwebhookconfigurations cert-manager-webhook -o 'jsonpath={.webhooks[0].clientConfig.caBundle}' + grep -q . + log 'probing cert-manager webhook' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T13:29:58+0000]' probing cert-manager webhook [2026-10-05T13:29:58+0000] probing cert-manager webhook + local probe_file ++ mktemp + probe_file=/tmp/tmp.67XTulhmWb + cat + local retries=0 + kubectl_bin apply -f /tmp/tmp.67XTulhmWb --dry-run=server + rm -f /tmp/tmp.67XTulhmWb + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/pitr/conf/issuer.yml ++ mktemp + local LAST_OUT=/tmp/tmp.jnzMSjZNUI ++ mktemp + local LAST_ERR=/tmp/tmp.k29BfIGftX + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/pitr/conf/issuer.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jnzMSjZNUI issuer.cert-manager.io/selfsigning-issuer created + cat /tmp/tmp.k29BfIGftX + rm /tmp/tmp.jnzMSjZNUI /tmp/tmp.k29BfIGftX + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/pitr/conf/cert.yml ++ mktemp + local LAST_OUT=/tmp/tmp.s2C5iWLAE7 ++ mktemp + local LAST_ERR=/tmp/tmp.4QMiJ04YKS + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/pitr/conf/cert.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.s2C5iWLAE7 certificate.cert-manager.io/tls-minio created + cat /tmp/tmp.4QMiJ04YKS + rm /tmp/tmp.s2C5iWLAE7 /tmp/tmp.4QMiJ04YKS + return 0 + kubectl_bin wait certificate/tls-minio --for=condition=Ready --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.mCeDoyvzrG ++ mktemp + local LAST_ERR=/tmp/tmp.7Q6XIRHO3a + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait certificate/tls-minio --for=condition=Ready --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mCeDoyvzrG certificate.cert-manager.io/tls-minio condition met + cat /tmp/tmp.7Q6XIRHO3a + rm /tmp/tmp.mCeDoyvzrG /tmp/tmp.7Q6XIRHO3a + return 0 + start_minio tls-minio + deploy_helm pitr-2340 + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" already exists with the same configuration, skipping + helm repo add minio https://charts.min.io/ "minio" already exists with the same configuration, skipping + helm repo update Hang tight while we grab the latest from your chart repositories... ...Successfully got an update from the "minio" chart repository ...Successfully got an update from the "hashicorp" chart repository ...Successfully got an update from the "percona" chart repository ...Successfully got an update from the "chaos-mesh" chart repository Update Complete. ⎈Happy Helming!⎈ + local cert_secret=tls-minio + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G') + local minio_args + [[ -n tls-minio ]] + endpoint=https://minio-service:9000 + minio_args+=(--set tls.enabled=true --set tls.certSecret="$cert_secret" --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key) + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: failed post-install: 1 error occurred: * timed out waiting for the condition + [[ 1 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 2 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 3 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 4 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 5 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 6 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 7 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 8 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 9 -ge 10 ]] + let n+=1 + sleep 60 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio Error: INSTALLATION FAILED: cannot re-use a name that is still in use + [[ 10 -ge 10 ]] + echo 'The command '\''helm' install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key 'minio/minio'\'' has failed after 10 attempts.' The command 'helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set users[0].accessKey=some-access-key --set users[0].secretKey=some-secret-key --set users[0].policy=consoleAdmin --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio' has failed after 10 attempts. + exit 1