Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/logs/pitr-physical-backup-source.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + main + create_infra pitr-physical-backup-source-352 + local ns=pitr-physical-backup-source-352 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.ksZVoK67aW ++ mktemp + local LAST_ERR=/tmp/tmp.QcXzwcuM7p + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ksZVoK67aW customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.QcXzwcuM7p + rm /tmp/tmp.ksZVoK67aW /tmp/tmp.QcXzwcuM7p + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.VvJqC8keBd ++ mktemp + local LAST_ERR=/tmp/tmp.l7dC0zhmWy + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VvJqC8keBd + cat /tmp/tmp.l7dC0zhmWy + rm /tmp/tmp.VvJqC8keBd /tmp/tmp.l7dC0zhmWy + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.eIJv7UZ762 ++ mktemp + local LAST_ERR=/tmp/tmp.qEDjoCujBo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eIJv7UZ762 + cat /tmp/tmp.qEDjoCujBo + rm /tmp/tmp.eIJv7UZ762 /tmp/tmp.qEDjoCujBo + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.C9K0FD1ejh ++ mktemp + local LAST_ERR=/tmp/tmp.CPMv8yKTew + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.C9K0FD1ejh + cat /tmp/tmp.CPMv8yKTew + rm /tmp/tmp.C9K0FD1ejh /tmp/tmp.CPMv8yKTew + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.wH0sL8yGwQ ++ mktemp + local LAST_ERR=/tmp/tmp.5qbLjWZRXh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wH0sL8yGwQ + cat /tmp/tmp.5qbLjWZRXh + rm /tmp/tmp.wH0sL8yGwQ /tmp/tmp.5qbLjWZRXh + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.rc1q7vUkjz ++ mktemp + local LAST_ERR=/tmp/tmp.jVMi4d1imb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rc1q7vUkjz clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.jVMi4d1imb + rm /tmp/tmp.rc1q7vUkjz /tmp/tmp.jVMi4d1imb + return 0 + check_crd_for_deletion PR-2315-13ff35222 + local git_tag=PR-2315-13ff35222 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2315-13ff35222/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Zb4jHIqB7y +++ mktemp ++ local LAST_ERR=/tmp/tmp.4UvRT4rU1d ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Zb4jHIqB7y ++ cat /tmp/tmp.4UvRT4rU1d Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Zb4jHIqB7y ++ cat /tmp/tmp.4UvRT4rU1d Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Zb4jHIqB7y ++ cat /tmp/tmp.4UvRT4rU1d Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.Zb4jHIqB7y ++ cat /tmp/tmp.4UvRT4rU1d Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.Zb4jHIqB7y /tmp/tmp.4UvRT4rU1d ++ return 1 + [[ '' == Terminating ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + xargs kubectl delete ns + desc 'cleaned up old namespaces psmdb-operator' ++ mktemp + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.oRDZN9uHp7 ++ mktemp + local LAST_OUT=/tmp/tmp.5oP9FJ4bVA ++ mktemp + local LAST_ERR=/tmp/tmp.FmklvQcgo8 + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.usWZoPinAv + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oRDZN9uHp7 + cat /tmp/tmp.usWZoPinAv + rm /tmp/tmp.oRDZN9uHp7 /tmp/tmp.usWZoPinAv + return 0 namespace "pitr-physical-backup-source-20390" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5oP9FJ4bVA namespace "psmdb-operator" deleted + cat /tmp/tmp.FmklvQcgo8 + rm /tmp/tmp.5oP9FJ4bVA /tmp/tmp.FmklvQcgo8 + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.lqPWuE3zAm ++ mktemp + local LAST_ERR=/tmp/tmp.Fzq5wY0nhA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lqPWuE3zAm + cat /tmp/tmp.Fzq5wY0nhA + rm /tmp/tmp.lqPWuE3zAm /tmp/tmp.Fzq5wY0nhA + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.feT8vO9gbZ ++ mktemp + local LAST_ERR=/tmp/tmp.dI0OGMp48T + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.feT8vO9gbZ namespace/psmdb-operator created + cat /tmp/tmp.dI0OGMp48T + rm /tmp/tmp.feT8vO9gbZ /tmp/tmp.dI0OGMp48T + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.ooVwDwNMy0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.bdeHFtkZXj ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ooVwDwNMy0 ++ cat /tmp/tmp.bdeHFtkZXj ++ rm /tmp/tmp.ooVwDwNMy0 /tmp/tmp.bdeHFtkZXj ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2315-13ff35222-29-cluster6 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.0PSglNiJU1 ++ mktemp + local LAST_ERR=/tmp/tmp.rjnopT48NK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2315-13ff35222-29-cluster6 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0PSglNiJU1 Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2315-13ff35222-29-cluster6" modified. + cat /tmp/tmp.rjnopT48NK + rm /tmp/tmp.0PSglNiJU1 /tmp/tmp.rjnopT48NK + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2315-13ff35222' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2315-13ff35222 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.fhYi5GmY2A ++ mktemp + local LAST_ERR=/tmp/tmp.T8DrCgaDDV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fhYi5GmY2A customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.T8DrCgaDDV + rm /tmp/tmp.fhYi5GmY2A /tmp/tmp.T8DrCgaDDV + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.og6CMgbusG ++ mktemp + local LAST_ERR=/tmp/tmp.pbSRhuTna4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.og6CMgbusG clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.pbSRhuTna4 + rm /tmp/tmp.og6CMgbusG /tmp/tmp.pbSRhuTna4 + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2315-13ff35222") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ZJbNu0cMw5 ++ mktemp + local LAST_ERR=/tmp/tmp.7ClqrKzVWI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZJbNu0cMw5 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.7ClqrKzVWI + rm /tmp/tmp.ZJbNu0cMw5 /tmp/tmp.7ClqrKzVWI + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.07Mww9b7AB +++ mktemp ++ local LAST_ERR=/tmp/tmp.OdeORhQLYP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.07Mww9b7AB ++ cat /tmp/tmp.OdeORhQLYP ++ rm /tmp/tmp.07Mww9b7AB /tmp/tmp.OdeORhQLYP ++ return 0 + wait_operator_pod percona-server-mongodb-operator-7b49d5666-hr8fb + local pod=percona-server-mongodb-operator-7b49d5666-hr8fb + set +o xtrace waiting for pod/percona-server-mongodb-operator-7b49d5666-hr8fb to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.VTSrlfXGF3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.6353f2iCY9 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VTSrlfXGF3 ++ cat /tmp/tmp.6353f2iCY9 ++ rm /tmp/tmp.VTSrlfXGF3 /tmp/tmp.6353f2iCY9 ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7b49d5666-hr8fb ++ mktemp + local LAST_OUT=/tmp/tmp.0oe5ga6nTo ++ mktemp + local LAST_ERR=/tmp/tmp.KuBnqxpGqJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-7b49d5666-hr8fb + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0oe5ga6nTo + cat /tmp/tmp.KuBnqxpGqJ + rm /tmp/tmp.0oe5ga6nTo /tmp/tmp.KuBnqxpGqJ + return 0 2026-06-22T10:58:30.438Z INFO setup Manager starting up {"gitCommit": "13ff35222ab7cb0709f1eef1197e12f4cb026db0", "gitBranch": "PR-2315-13ff35222", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} + create_namespace pitr-physical-backup-source-352 + local namespace=pitr-physical-backup-source-352 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces pitr-physical-backup-source-352' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pitr-physical-backup-source-352 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pitr-physical-backup-source-352 --ignore-not-found ++ mktemp + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.yhce97RAYn + local LAST_OUT=/tmp/tmp.Hh4cJqKeIu ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.f7PPx5jHRQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.hZLVR6b18d + local exit_status=0 + local timeout=4 + for i in $(seq 0 2) + set +e + kubectl delete namespace pitr-physical-backup-source-352 --ignore-not-found ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Hh4cJqKeIu + cat /tmp/tmp.hZLVR6b18d + rm /tmp/tmp.Hh4cJqKeIu /tmp/tmp.hZLVR6b18d + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yhce97RAYn + cat /tmp/tmp.f7PPx5jHRQ + rm /tmp/tmp.yhce97RAYn /tmp/tmp.f7PPx5jHRQ + return 0 + kubectl_bin wait --for=delete namespace pitr-physical-backup-source-352 ++ mktemp + local LAST_OUT=/tmp/tmp.0d2KNXFbIL ++ mktemp + local LAST_ERR=/tmp/tmp.RvF02ugDeo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace pitr-physical-backup-source-352 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0d2KNXFbIL + cat /tmp/tmp.RvF02ugDeo + rm /tmp/tmp.0d2KNXFbIL /tmp/tmp.RvF02ugDeo + return 0 + desc 'create namespace pitr-physical-backup-source-352' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pitr-physical-backup-source-352 ----------------------------------------------------------------------------------- + kubectl_bin create namespace pitr-physical-backup-source-352 ++ mktemp + local LAST_OUT=/tmp/tmp.ps3raFHBHA ++ mktemp + local LAST_ERR=/tmp/tmp.ZYFwVWqDqj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pitr-physical-backup-source-352 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ps3raFHBHA namespace/pitr-physical-backup-source-352 created + cat /tmp/tmp.ZYFwVWqDqj + rm /tmp/tmp.ps3raFHBHA /tmp/tmp.ZYFwVWqDqj + return 0 + set_kube_ctx pitr-physical-backup-source-352 + local namespace=pitr-physical-backup-source-352 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.omPEKpVKQm +++ mktemp ++ local LAST_ERR=/tmp/tmp.GFlw40jvSE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.omPEKpVKQm ++ cat /tmp/tmp.GFlw40jvSE ++ rm /tmp/tmp.omPEKpVKQm /tmp/tmp.GFlw40jvSE ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2315-13ff35222-29-cluster6 --namespace=pitr-physical-backup-source-352 ++ mktemp + local LAST_OUT=/tmp/tmp.txfeCO67Sy ++ mktemp + local LAST_ERR=/tmp/tmp.YpeaaPSAVT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2315-13ff35222-29-cluster6 --namespace=pitr-physical-backup-source-352 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.txfeCO67Sy Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2315-13ff35222-29-cluster6" modified. + cat /tmp/tmp.YpeaaPSAVT + rm /tmp/tmp.txfeCO67Sy /tmp/tmp.YpeaaPSAVT + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Mon Jun 22 10:59:09 2026 NAMESPACE: pitr-physical-backup-source-352 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.pitr-physical-backup-source-352.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace pitr-physical-backup-source-352 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace pitr-physical-backup-source-352 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-physical-backup-source-352 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-physical-backup-source-352 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f0JrGq3BmQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.UYx8Rr1yJ8 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.f0JrGq3BmQ ++ cat /tmp/tmp.UYx8Rr1yJ8 ++ rm /tmp/tmp.f0JrGq3BmQ /tmp/tmp.UYx8Rr1yJ8 ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-s7fnh + wait_pod minio-service-6d5f646cdc-s7fnh + local pod=minio-service-6d5f646cdc-s7fnh + set +o xtrace waiting for pod/minio-service-6d5f646cdc-s7fnh to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-backup-source-352.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.MoOJFNYtaJ ++ mktemp + local LAST_ERR=/tmp/tmp.gj9tpgwygx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MoOJFNYtaJ make_bucket: operator-testing pod "aws-cli" deleted from pitr-physical-backup-source-352 namespace + cat /tmp/tmp.gj9tpgwygx All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_pitr-physical-backup-source-352 + rm /tmp/tmp.MoOJFNYtaJ /tmp/tmp.gj9tpgwygx + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/conf/minio-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.lTQryMaIrr ++ mktemp + local LAST_ERR=/tmp/tmp.SAS3fyp9ma + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/conf/minio-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lTQryMaIrr secret/some-users created deployment.apps/psmdb-client created secret/minio-secret created + cat /tmp/tmp.SAS3fyp9ma + rm /tmp/tmp.lTQryMaIrr /tmp/tmp.SAS3fyp9ma + return 0 + cluster=some-name + desc 'create first PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster some-name ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2315-13ff35222"' + /usr/sbin/sed -e s/NAME_SPACE/pitr-physical-backup-source-352/g + local LAST_OUT=/tmp/tmp.1oDD6KXj3O + yq eval '(.spec | select(has("search"))).search.image = "mongodb/mongodb-community-search:latest"' ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.oapk4iFXGC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1oDD6KXj3O perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.oapk4iFXGC + rm /tmp/tmp.1oDD6KXj3O /tmp/tmp.oapk4iFXGC + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready....OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JUIz5m1iOr +++ mktemp ++ local LAST_ERR=/tmp/tmp.As46oSlE9u ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JUIz5m1iOr ++ cat /tmp/tmp.As46oSlE9u ++ rm /tmp/tmp.JUIz5m1iOr /tmp/tmp.As46oSlE9u ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.....OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FcD833j1tF +++ mktemp ++ local LAST_ERR=/tmp/tmp.h8QQaa4dhK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FcD833j1tF ++ cat /tmp/tmp.h8QQaa4dhK ++ rm /tmp/tmp.FcD833j1tF /tmp/tmp.h8QQaa4dhK ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8rFJnJb99o +++ mktemp ++ local LAST_ERR=/tmp/tmp.cflkO7Offw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8rFJnJb99o ++ cat /tmp/tmp.cflkO7Offw ++ rm /tmp/tmp.8rFJnJb99o /tmp/tmp.cflkO7Offw ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness.................................... + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9uRKcz7iKi +++ mktemp ++ local LAST_ERR=/tmp/tmp.6es6r2njPx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9uRKcz7iKi ++ cat /tmp/tmp.6es6r2njPx ++ rm /tmp/tmp.9uRKcz7iKi /tmp/tmp.6es6r2njPx ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zpdo3PSI2B +++ mktemp ++ local LAST_ERR=/tmp/tmp.WQ2v7lLrbL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zpdo3PSI2B ++ cat /tmp/tmp.WQ2v7lLrbL ++ rm /tmp/tmp.zpdo3PSI2B /tmp/tmp.WQ2v7lLrbL ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sEVlcj0Yj6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.5UpziDoqXw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.sEVlcj0Yj6 ++ cat /tmp/tmp.5UpziDoqXw ++ rm /tmp/tmp.sEVlcj0Yj6 /tmp/tmp.5UpziDoqXw ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + sleep 10 + write_initial_data + desc 'create user myApp' + set +o xtrace ----------------------------------------------------------------------------------- create user myApp ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-352 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XxVqr86LbO +++ mktemp ++ local LAST_ERR=/tmp/tmp.pOerbyx4tL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XxVqr86LbO ++ cat /tmp/tmp.pOerbyx4tL ++ rm /tmp/tmp.XxVqr86LbO /tmp/tmp.pOerbyx4tL ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.ThBRxIrLnN ++ mktemp + local LAST_ERR=/tmp/tmp.B8Y1r1pHCt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ThBRxIrLnN Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("c5d5b83c-1160-4482-9296-06a5a974fa52") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.B8Y1r1pHCt + rm /tmp/tmp.ThBRxIrLnN /tmp/tmp.B8Y1r1pHCt + return 0 + sleep 2 + write_document + local cmp_postfix= + local sleep_value=0 + log 'write initial data, read from all' + set +o xtrace [2026-06-22T11:03:02+0000] write initial data, read from all + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.meckb9hrpQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.oM2DuyJlx1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.meckb9hrpQ ++ cat /tmp/tmp.oM2DuyJlx1 ++ rm /tmp/tmp.meckb9hrpQ /tmp/tmp.oM2DuyJlx1 ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.aAutVFteBT ++ mktemp + local LAST_ERR=/tmp/tmp.AcmHEZLCvU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aAutVFteBT Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("17882566-b966-4e6f-9da6-37b01f173bee") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.AcmHEZLCvU + rm /tmp/tmp.aAutVFteBT /tmp/tmp.AcmHEZLCvU + return 0 + sleep 0 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-06-22T11:03:05+0000] running db.test.command() in myApp + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JSYF5jqMSv +++ mktemp ++ local LAST_ERR=/tmp/tmp.ph25p7WFh7 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JSYF5jqMSv ++ cat /tmp/tmp.ph25p7WFh7 ++ rm /tmp/tmp.JSYF5jqMSv /tmp/tmp.ph25p7WFh7 ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.1fQglSkmaV ++ mktemp + local LAST_ERR=/tmp/tmp.8H1aOQuxa1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1fQglSkmaV + cat /tmp/tmp.8H1aOQuxa1 + rm /tmp/tmp.1fQglSkmaV /tmp/tmp.8H1aOQuxa1 + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.JwNUXLpdvn/find + wait_backup_agent some-name-rs0-0 + local agent_pod=some-name-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-0...2026-06-22T11:02:26.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-1 + local agent_pod=some-name-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-1...2026-06-22T11:02:27.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-2 + local agent_pod=some-name-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-2...2026-06-22T11:02:30.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-0 + local agent_pod=some-name-rs1-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-0...2026-06-22T11:02:23.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-1 + local agent_pod=some-name-rs1-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-1...2026-06-22T11:02:28.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-2 + local agent_pod=some-name-rs1-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-2...2026-06-22T11:02:31.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-0 + local agent_pod=some-name-rs2-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-0...2026-06-22T11:02:24.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-1 + local agent_pod=some-name-rs2-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-1...2026-06-22T11:02:23.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-2 + local agent_pod=some-name-rs2-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-2...2026-06-22T11:02:31.000+0000 I listening for the commands + echo 'Sleeping for 360 seconds' Sleeping for 360 seconds + sleep 360 + backup_name_minio=backup-minio + desc 'restore pitr type date using backupSource' + set +o xtrace ----------------------------------------------------------------------------------- restore pitr type date using backupSource ----------------------------------------------------------------------------------- + run_backup backup-minio 1 physical + local name=backup-minio + local idx=1 + local type=physical + desc 'run backup backup-minio-1' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-1 ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml + /usr/sbin/sed -e 's/name:/name: backup-minio-1/' + kubectl_bin apply -f - + /usr/sbin/sed -e 's/type:/type: physical/' ++ mktemp + local LAST_OUT=/tmp/tmp.aE50tY3lZF ++ mktemp + local LAST_ERR=/tmp/tmp.pliRAPMRrn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aE50tY3lZF perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created + cat /tmp/tmp.pliRAPMRrn + rm /tmp/tmp.aE50tY3lZF /tmp/tmp.pliRAPMRrn + return 0 + wait_backup backup-minio-1 + local backup_name=backup-minio-1 + local target_state=ready + set +o xtrace waiting for backup-minio-1 to reach ready state.........OK + compare_latest_restorable_time some-name-rs0 backup-minio-1 + local cluster=some-name-rs0 + local backup_name=backup-minio-1 + local latest_restorable_time + local backup_time ++ get_latest_restorable_time some-name-rs0 ++ local cluster=some-name-rs0 ++ local first_timestamp ++ local second_timestamp ++ local retry=0 ++ [[ '' != '' ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.SjLBO0gA68 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OfA3wE7DfP +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.SjLBO0gA68 +++ cat /tmp/tmp.OfA3wE7DfP +++ rm /tmp/tmp.SjLBO0gA68 /tmp/tmp.OfA3wE7DfP +++ return 0 ++ first_timestamp=null ++ sleep 5 ++ [[ null != '' ]] ++ [[ null != null ]] ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ null != '' ]] ++ [[ null != null ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KbXUAYa6bF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.AMdriAe2py +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.KbXUAYa6bF +++ cat /tmp/tmp.AMdriAe2py +++ rm /tmp/tmp.KbXUAYa6bF /tmp/tmp.AMdriAe2py +++ return 0 ++ first_timestamp=null ++ sleep 5 ++ [[ null != '' ]] ++ [[ null != null ]] ++ let retry+=1 ++ [[ 2 -gt 30 ]] ++ [[ null != '' ]] ++ [[ null != null ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7qvKylWyFc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UbQjgNUUqA +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.7qvKylWyFc +++ cat /tmp/tmp.UbQjgNUUqA +++ rm /tmp/tmp.7qvKylWyFc /tmp/tmp.UbQjgNUUqA +++ return 0 ++ first_timestamp=1782126585 ++ sleep 5 ++ [[ 1782126585 != '' ]] ++ [[ 1782126585 != null ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sFszh2XcTA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.eR1s70XHur +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.sFszh2XcTA +++ cat /tmp/tmp.eR1s70XHur +++ rm /tmp/tmp.sFszh2XcTA /tmp/tmp.eR1s70XHur +++ return 0 ++ second_timestamp=1782126585 ++ let retry+=1 ++ [[ 3 -gt 30 ]] ++ [[ 1782126585 != '' ]] ++ [[ 1782126585 != null ]] ++ [[ 1782126585 == 1782126585 ]] ++ /usr/sbin/date -u -d @1782126585 +%Y-%m-%dT%H:%M:%SZ + latest_restorable_time=2026-06-22T11:09:45Z ++ get_latest_restorable_time_from_backup_object backup-minio-1 ++ local backup_name=backup-minio-1 ++ local latestRestorableTime ++ local retry=0 ++ [[ '' != '' ]] ++ sleep 5 +++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.K0aWUXazjM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JSRHhrep7J +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.K0aWUXazjM +++ cat /tmp/tmp.JSRHhrep7J +++ rm /tmp/tmp.K0aWUXazjM /tmp/tmp.JSRHhrep7J +++ return 0 ++ latestRestorableTime=2026-06-22T11:09:45Z ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 2026-06-22T11:09:45Z != '' ]] ++ [[ 2026-06-22T11:09:45Z != null ]] ++ echo 2026-06-22T11:09:45Z + backup_time=2026-06-22T11:09:45Z + [[ 2026-06-22T11:09:45Z != 2026\-06\-22T11\:09\:45Z ]] + reset_collection + desc 'reset data' + set +o xtrace ----------------------------------------------------------------------------------- reset data ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.remove({})' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local 'command=use myApp\n db.test.remove({})' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.klYlVEzEtd +++ mktemp ++ local LAST_ERR=/tmp/tmp.zDfUhDXtFN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.klYlVEzEtd ++ cat /tmp/tmp.zDfUhDXtFN ++ rm /tmp/tmp.klYlVEzEtd /tmp/tmp.zDfUhDXtFN ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.xtrmvGZwsl ++ mktemp + local LAST_ERR=/tmp/tmp.KBt2lJjUSh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xtrmvGZwsl Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("3c11c634-c169-4576-9c81-c6a3b0ab5a19") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nRemoved" : 1 }) bye + cat /tmp/tmp.KBt2lJjUSh + rm /tmp/tmp.xtrmvGZwsl /tmp/tmp.KBt2lJjUSh + return 0 + sleep 2 + write_document '' 120 + local cmp_postfix= + local sleep_value=120 + log 'write initial data, read from all' + set +o xtrace [2026-06-22T11:10:14+0000] write initial data, read from all + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RuETeYeBG1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.K395DTE48T ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RuETeYeBG1 ++ cat /tmp/tmp.K395DTE48T ++ rm /tmp/tmp.RuETeYeBG1 /tmp/tmp.K395DTE48T ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.3EIOQB4BSP ++ mktemp + local LAST_ERR=/tmp/tmp.s71tWsMrcW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3EIOQB4BSP Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("529199d3-d0da-4780-8e50-78f2aa196948") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.s71tWsMrcW + rm /tmp/tmp.3EIOQB4BSP /tmp/tmp.s71tWsMrcW + return 0 + sleep 120 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-06-22T11:12:17+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 mongodb '' '' 27017 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mQjLPozFdc +++ mktemp ++ local LAST_ERR=/tmp/tmp.QTdw67YNEB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mQjLPozFdc ++ cat /tmp/tmp.QTdw67YNEB ++ rm /tmp/tmp.mQjLPozFdc /tmp/tmp.QTdw67YNEB ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.0IatNgbzNK ++ mktemp + local LAST_ERR=/tmp/tmp.eCyEgdGLxy + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0IatNgbzNK + cat /tmp/tmp.eCyEgdGLxy + rm /tmp/tmp.0IatNgbzNK /tmp/tmp.eCyEgdGLxy + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.JwNUXLpdvn/find ++ run_mongos 'new Date().getTime() / 1000' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 mongodb '' --quiet ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match' ++ local 'command=new Date().getTime() / 1000' ++ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ cut -d. -f1 ++ local mongo_flag=--quiet ++ local port=27017 ++ local mongo_bin=mongo +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KSuaPwD1Ve ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8oE3JRBnCT +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.KSuaPwD1Ve +++ cat /tmp/tmp.8oE3JRBnCT +++ rm /tmp/tmp.KSuaPwD1Ve /tmp/tmp.8oE3JRBnCT +++ return 0 ++ local client_container=psmdb-client-bb8b97679-82f4r ++ kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BHl0QriXjS +++ mktemp ++ local LAST_ERR=/tmp/tmp.eX3QsgDmcX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BHl0QriXjS ++ cat /tmp/tmp.eX3QsgDmcX ++ rm /tmp/tmp.BHl0QriXjS /tmp/tmp.eX3QsgDmcX ++ return 0 + time_now=1782126742 + check_recovery backup-minio-1 date 1782126742 '' some-name backupSource + local backup_name=backup-minio-1 + local restore_type=date + local restore_date=1782126742 + local cmp_postfix= + local cluster_name=some-name + local backupSource=backupSource ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7C22kdLz33 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KgNRiMG7Nk +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.7C22kdLz33 +++ cat /tmp/tmp.KgNRiMG7Nk +++ rm /tmp/tmp.7C22kdLz33 /tmp/tmp.KgNRiMG7Nk +++ return 0 ++ echo 1782126714 + local latest_ts=1782126714 + desc 'write more data before restore by date' + set +o xtrace ----------------------------------------------------------------------------------- write more data before restore by date ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nHksXapCKV +++ mktemp ++ local LAST_ERR=/tmp/tmp.s6bUAMsz4m ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nHksXapCKV ++ cat /tmp/tmp.s6bUAMsz4m ++ rm /tmp/tmp.nHksXapCKV /tmp/tmp.s6bUAMsz4m ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.CQFx4rldQW ++ mktemp + local LAST_ERR=/tmp/tmp.tsjcOeQaiK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CQFx4rldQW Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("3a4b3c7d-2e87-46b8-9097-8e28d957ed18") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.tsjcOeQaiK + rm /tmp/tmp.CQFx4rldQW /tmp/tmp.tsjcOeQaiK + return 0 + [[ -n 1782126742 ]] ++ format_date 1782126742 ++ local timestamp=1782126742 +++ TZ=UTC +++ /usr/sbin/date -d@1782126742 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:12:22 + desc 'Restoring to time 2026-06-22 11:12:22' + set +o xtrace ----------------------------------------------------------------------------------- Restoring to time 2026-06-22 11:12:22 ----------------------------------------------------------------------------------- + retries=0 + [[ 1782126714 -gt 1782126742 ]] + [[ 0 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vAJD0tITo3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.njoo6CYtfF +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.vAJD0tITo3 +++ cat /tmp/tmp.njoo6CYtfF +++ rm /tmp/tmp.vAJD0tITo3 /tmp/tmp.njoo6CYtfF +++ return 0 ++ echo 1782126714 + latest_ts=1782126714 + retries=1 ++ format_date 1782126714 ++ local timestamp=1782126714 +++ TZ=UTC +++ /usr/sbin/date -d@1782126714 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:11:54 ++ format_date 1782126742 ++ local timestamp=1782126742 +++ TZ=UTC +++ /usr/sbin/date -d@1782126742 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:12:22 + echo 'Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22)' Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22) + sleep 10 + [[ 1782126714 -gt 1782126742 ]] + [[ 1 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rdCIaZWbqj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Xzty46Ri7u +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.rdCIaZWbqj +++ cat /tmp/tmp.Xzty46Ri7u +++ rm /tmp/tmp.rdCIaZWbqj /tmp/tmp.Xzty46Ri7u +++ return 0 ++ echo 1782126714 + latest_ts=1782126714 + retries=2 ++ format_date 1782126714 ++ local timestamp=1782126714 +++ TZ=UTC +++ /usr/sbin/date -d@1782126714 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:11:54 ++ format_date 1782126742 ++ local timestamp=1782126742 +++ TZ=UTC +++ /usr/sbin/date -d@1782126742 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:12:22 + echo 'Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22)' Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22) + sleep 10 + [[ 1782126714 -gt 1782126742 ]] + [[ 2 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.paUnyTUUN6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zNLI6FqRDO +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.paUnyTUUN6 +++ cat /tmp/tmp.zNLI6FqRDO +++ rm /tmp/tmp.paUnyTUUN6 /tmp/tmp.zNLI6FqRDO +++ return 0 ++ echo 1782126714 + latest_ts=1782126714 + retries=3 ++ format_date 1782126714 ++ local timestamp=1782126714 +++ TZ=UTC +++ /usr/sbin/date -d@1782126714 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:11:54 ++ format_date 1782126742 ++ local timestamp=1782126742 +++ TZ=UTC +++ /usr/sbin/date -d@1782126742 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:12:22 + echo 'Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22)' Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22) + sleep 10 + [[ 1782126714 -gt 1782126742 ]] + [[ 3 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Jj4Q3KfvGN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VYeo3Yr2Em +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Jj4Q3KfvGN +++ cat /tmp/tmp.VYeo3Yr2Em +++ rm /tmp/tmp.Jj4Q3KfvGN /tmp/tmp.VYeo3Yr2Em +++ return 0 ++ echo 1782126714 + latest_ts=1782126714 + retries=4 ++ format_date 1782126714 ++ local timestamp=1782126714 +++ TZ=UTC +++ /usr/sbin/date -d@1782126714 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:11:54 ++ format_date 1782126742 ++ local timestamp=1782126742 +++ TZ=UTC +++ /usr/sbin/date -d@1782126742 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:12:22 + echo 'Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22)' Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22) + sleep 10 + [[ 1782126714 -gt 1782126742 ]] + [[ 4 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3RUuVkMA2f ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JIuS44LG3O +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.3RUuVkMA2f +++ cat /tmp/tmp.JIuS44LG3O +++ rm /tmp/tmp.3RUuVkMA2f /tmp/tmp.JIuS44LG3O +++ return 0 ++ echo 1782126714 + latest_ts=1782126714 + retries=5 ++ format_date 1782126714 ++ local timestamp=1782126714 +++ TZ=UTC +++ /usr/sbin/date -d@1782126714 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:11:54 ++ format_date 1782126742 ++ local timestamp=1782126742 +++ TZ=UTC +++ /usr/sbin/date -d@1782126742 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:12:22 + echo 'Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22)' Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22) + sleep 10 + [[ 1782126714 -gt 1782126742 ]] + [[ 5 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IvrZQiGtl5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xuHiDsnCUO +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.IvrZQiGtl5 +++ cat /tmp/tmp.xuHiDsnCUO +++ rm /tmp/tmp.IvrZQiGtl5 /tmp/tmp.xuHiDsnCUO +++ return 0 ++ echo 1782126714 + latest_ts=1782126714 + retries=6 ++ format_date 1782126714 ++ local timestamp=1782126714 +++ TZ=UTC +++ /usr/sbin/date -d@1782126714 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:11:54 ++ format_date 1782126742 ++ local timestamp=1782126742 +++ TZ=UTC +++ /usr/sbin/date -d@1782126742 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:12:22 + echo 'Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22)' Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22) + sleep 10 + [[ 1782126714 -gt 1782126742 ]] + [[ 6 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.A0l2svyYzL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Fy8aOezpHp +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.A0l2svyYzL +++ cat /tmp/tmp.Fy8aOezpHp +++ rm /tmp/tmp.A0l2svyYzL /tmp/tmp.Fy8aOezpHp +++ return 0 ++ echo 1782126714 + latest_ts=1782126714 + retries=7 ++ format_date 1782126714 ++ local timestamp=1782126714 +++ TZ=UTC +++ /usr/sbin/date -d@1782126714 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:11:54 ++ format_date 1782126742 ++ local timestamp=1782126742 +++ TZ=UTC +++ /usr/sbin/date -d@1782126742 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:12:22 + echo 'Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22)' Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22) + sleep 10 + [[ 1782126714 -gt 1782126742 ]] + [[ 7 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json ++++ mktemp +++ local LAST_OUT=/tmp/tmp.e2phJqTg4O ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kPEbpsZE9u +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.e2phJqTg4O +++ cat /tmp/tmp.kPEbpsZE9u +++ rm /tmp/tmp.e2phJqTg4O /tmp/tmp.kPEbpsZE9u +++ return 0 ++ echo 1782126714 + latest_ts=1782126714 + retries=8 ++ format_date 1782126714 ++ local timestamp=1782126714 +++ TZ=UTC +++ /usr/sbin/date -d@1782126714 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:11:54 ++ format_date 1782126742 ++ local timestamp=1782126742 +++ TZ=UTC +++ /usr/sbin/date -d@1782126742 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:12:22 + echo 'Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22)' Waiting for last oplog chunk (2026-06-22 11:11:54) to be greater than restore target (2026-06-22 11:12:22) + sleep 10 + [[ 1782126714 -gt 1782126742 ]] + [[ 8 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4parBDGm8l ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Gub3vlaBzl +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.4parBDGm8l +++ cat /tmp/tmp.Gub3vlaBzl +++ rm /tmp/tmp.4parBDGm8l /tmp/tmp.Gub3vlaBzl +++ return 0 ++ echo 1782126722 + latest_ts=1782126722 + retries=9 ++ format_date 1782126722 ++ local timestamp=1782126722 +++ TZ=UTC +++ /usr/sbin/date -d@1782126722 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:12:02 ++ format_date 1782126742 ++ local timestamp=1782126742 +++ TZ=UTC +++ /usr/sbin/date -d@1782126742 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:12:22 + echo 'Waiting for last oplog chunk (2026-06-22 11:12:02) to be greater than restore target (2026-06-22 11:12:22)' Waiting for last oplog chunk (2026-06-22 11:12:02) to be greater than restore target (2026-06-22 11:12:22) + sleep 10 + [[ 1782126722 -gt 1782126742 ]] + [[ 9 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CMp5OFxvLo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sRPYpc53Z1 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.CMp5OFxvLo +++ cat /tmp/tmp.sRPYpc53Z1 +++ rm /tmp/tmp.CMp5OFxvLo /tmp/tmp.sRPYpc53Z1 +++ return 0 ++ echo 1782126834 + latest_ts=1782126834 + retries=10 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 ++ format_date 1782126742 ++ local timestamp=1782126742 +++ TZ=UTC +++ /usr/sbin/date -d@1782126742 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:12:22 + echo 'Waiting for last oplog chunk (2026-06-22 11:13:54) to be greater than restore target (2026-06-22 11:12:22)' Waiting for last oplog chunk (2026-06-22 11:13:54) to be greater than restore target (2026-06-22 11:12:22) + sleep 10 + [[ 1782126834 -gt 1782126742 ]] + '[' -z backupSource ']' + desc 'check restore by date backupSource' + set +o xtrace ----------------------------------------------------------------------------------- check restore by date backupSource ----------------------------------------------------------------------------------- ++ get_backup_dest backup-minio-1 ++ local backup_name=backup-minio-1 ++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' +++ mktemp ++ sed 's|gs://||' ++ sed 's|s3://||' ++ local LAST_OUT=/tmp/tmp.T79AoWMt2R ++ sed 's|azure://||' +++ mktemp ++ local LAST_ERR=/tmp/tmp.sDyveTnbnt ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.T79AoWMt2R ++ cat /tmp/tmp.sDyveTnbnt ++ rm /tmp/tmp.T79AoWMt2R /tmp/tmp.sDyveTnbnt ++ return 0 + backup_dest=operator-testing/2026-06-22T11:09:23Z + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-1/' + /usr/sbin/sed -e /backupName/d + /usr/sbin/sed -e 's/pitrType:/type: date/' + kubectl_bin apply -f - + '[' -z 1782126742 ']' + /usr/sbin/sed -e 's|DESTINATION|operator-testing/2026-06-22T11:09:23Z|' ++ mktemp ++ format_date 1782126742 ++ local timestamp=1782126742 +++ TZ=UTC +++ /usr/sbin/date -d@1782126742 '+%Y-%m-%d %H:%M:%S' + '[' -n '' ']' + yq ++ echo 2026-06-22 11:12:22 + local LAST_OUT=/tmp/tmp.oURZHiaN3E + /usr/sbin/sed -e 's/date:/date: 2026-06-22 11:12:22/' ++ get_bucket_name backup-minio-1 ++ local backup_name=backup-minio-1 ++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}' ++ mktemp +++ mktemp + local LAST_ERR=/tmp/tmp.jcMJW77ojN + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ local LAST_OUT=/tmp/tmp.PWqiPKKL7N +++ mktemp + for i in $(seq 0 2) + set +e + kubectl apply -f - ++ local LAST_ERR=/tmp/tmp.mu2IC30nVT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PWqiPKKL7N ++ cat /tmp/tmp.mu2IC30nVT ++ rm /tmp/tmp.PWqiPKKL7N /tmp/tmp.mu2IC30nVT ++ return 0 + /usr/sbin/sed -e 's|BUCKET-NAME|operator-testing|' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oURZHiaN3E perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created + cat /tmp/tmp.jcMJW77ojN + rm /tmp/tmp.oURZHiaN3E /tmp/tmp.jcMJW77ojN + return 0 + wait_restore backup-minio-1 some-name requested 0 1200 + local backup_name=backup-minio-1 + local cluster_name=some-name + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=1200 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK Waiting psmdb-restore/restore-backup-minio-1 to reach state "requested" ........OK after 7 minutes + [[ 0 -eq 1 ]] + echo + wait_restore backup-minio-1 some-name ready 0 1600 + local backup_name=backup-minio-1 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=0 + local wait_time=1600 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK Waiting psmdb-restore/restore-backup-minio-1 to reach state "ready" ...OK after 2 minutes + [[ 0 -eq 1 ]] + echo + set -o xtrace + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ToDV3m6AK5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gOSpMqkgAd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ToDV3m6AK5 ++ cat /tmp/tmp.gOSpMqkgAd ++ rm /tmp/tmp.ToDV3m6AK5 /tmp/tmp.gOSpMqkgAd ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uYiKyBvEg5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.auTE5lDdgA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uYiKyBvEg5 ++ cat /tmp/tmp.auTE5lDdgA ++ rm /tmp/tmp.uYiKyBvEg5 /tmp/tmp.auTE5lDdgA ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wKpDdbh9Km +++ mktemp ++ local LAST_ERR=/tmp/tmp.oC6tsenaW1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wKpDdbh9Km ++ cat /tmp/tmp.oC6tsenaW1 ++ rm /tmp/tmp.wKpDdbh9Km /tmp/tmp.oC6tsenaW1 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness........................................................ + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SZMsSktsZx +++ mktemp ++ local LAST_ERR=/tmp/tmp.kEeX0vlHzN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.SZMsSktsZx ++ cat /tmp/tmp.kEeX0vlHzN ++ rm /tmp/tmp.SZMsSktsZx /tmp/tmp.kEeX0vlHzN ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.b3KPNwNolG +++ mktemp ++ local LAST_ERR=/tmp/tmp.oxnItQAR5j ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.b3KPNwNolG ++ cat /tmp/tmp.oxnItQAR5j ++ rm /tmp/tmp.b3KPNwNolG /tmp/tmp.oxnItQAR5j ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rPdiql9JZC +++ mktemp ++ local LAST_ERR=/tmp/tmp.hEmYb4txTS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rPdiql9JZC ++ cat /tmp/tmp.hEmYb4txTS ++ rm /tmp/tmp.rPdiql9JZC /tmp/tmp.hEmYb4txTS ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Lns7l6pulr +++ mktemp ++ local LAST_ERR=/tmp/tmp.cMtByBUHyL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Lns7l6pulr ++ cat /tmp/tmp.cMtByBUHyL ++ rm /tmp/tmp.Lns7l6pulr /tmp/tmp.cMtByBUHyL ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.c0SIXV4S1G +++ mktemp ++ local LAST_ERR=/tmp/tmp.VjjVoUmxhD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.c0SIXV4S1G ++ cat /tmp/tmp.VjjVoUmxhD ++ rm /tmp/tmp.c0SIXV4S1G /tmp/tmp.VjjVoUmxhD ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yyVs2aj7bt +++ mktemp ++ local LAST_ERR=/tmp/tmp.yuZMzFmO27 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yyVs2aj7bt ++ cat /tmp/tmp.yuZMzFmO27 ++ rm /tmp/tmp.yyVs2aj7bt /tmp/tmp.yuZMzFmO27 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness + sleep 10 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 '' + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-06-22T11:27:20+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qjdZvVj5R4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.8cF94RezlJ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qjdZvVj5R4 ++ cat /tmp/tmp.8cF94RezlJ ++ rm /tmp/tmp.qjdZvVj5R4 /tmp/tmp.8cF94RezlJ ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.blTkgaRxtr ++ mktemp + local LAST_ERR=/tmp/tmp.WGGkXyCzEE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.blTkgaRxtr + cat /tmp/tmp.WGGkXyCzEE + rm /tmp/tmp.blTkgaRxtr /tmp/tmp.WGGkXyCzEE + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.JwNUXLpdvn/find + desc 'delete PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- delete PSMDB cluster some-name ----------------------------------------------------------------------------------- + kubectl_bin delete psmdb some-name ++ mktemp + local LAST_OUT=/tmp/tmp.75LPLwdAbI ++ mktemp + local LAST_ERR=/tmp/tmp.we2HpZ9vSf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb some-name + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.75LPLwdAbI perconaservermongodb.psmdb.percona.com "some-name" deleted from pitr-physical-backup-source-352 namespace + cat /tmp/tmp.we2HpZ9vSf + rm /tmp/tmp.75LPLwdAbI /tmp/tmp.we2HpZ9vSf + return 0 + kubectl_bin delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.dBryesfT92 ++ mktemp + local LAST_ERR=/tmp/tmp.KTyZQK6HnG + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dBryesfT92 persistentvolumeclaim "mongod-data-some-name-cfg-0" deleted from pitr-physical-backup-source-352 namespace persistentvolumeclaim "mongod-data-some-name-cfg-1" deleted from pitr-physical-backup-source-352 namespace persistentvolumeclaim "mongod-data-some-name-cfg-2" deleted from pitr-physical-backup-source-352 namespace persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted from pitr-physical-backup-source-352 namespace persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted from pitr-physical-backup-source-352 namespace persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted from pitr-physical-backup-source-352 namespace persistentvolumeclaim "mongod-data-some-name-rs1-0" deleted from pitr-physical-backup-source-352 namespace persistentvolumeclaim "mongod-data-some-name-rs1-1" deleted from pitr-physical-backup-source-352 namespace persistentvolumeclaim "mongod-data-some-name-rs1-2" deleted from pitr-physical-backup-source-352 namespace persistentvolumeclaim "mongod-data-some-name-rs2-0" deleted from pitr-physical-backup-source-352 namespace persistentvolumeclaim "mongod-data-some-name-rs2-1" deleted from pitr-physical-backup-source-352 namespace persistentvolumeclaim "mongod-data-some-name-rs2-2" deleted from pitr-physical-backup-source-352 namespace + cat /tmp/tmp.KTyZQK6HnG + rm /tmp/tmp.dBryesfT92 /tmp/tmp.KTyZQK6HnG + return 0 + sleep 10 + desc 'recreate PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- recreate PSMDB cluster some-name ----------------------------------------------------------------------------------- + desc 'create second PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- create second PSMDB cluster some-name ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2315-13ff35222"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '(.spec | select(has("search"))).search.image = "mongodb/mongodb-community-search:latest"' + /usr/sbin/sed -e s/NAME_SPACE/pitr-physical-backup-source-352/g + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.XCPMCeqk3W ++ mktemp + local LAST_ERR=/tmp/tmp.Zizsm1rcKn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XCPMCeqk3W perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.Zizsm1rcKn + rm /tmp/tmp.XCPMCeqk3W /tmp/tmp.Zizsm1rcKn + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.......OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready......OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AEqoyGMrRG +++ mktemp ++ local LAST_ERR=/tmp/tmp.aM5FALdEYt ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AEqoyGMrRG ++ cat /tmp/tmp.aM5FALdEYt ++ rm /tmp/tmp.AEqoyGMrRG /tmp/tmp.aM5FALdEYt ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready....OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8jmV7BpeLU +++ mktemp ++ local LAST_ERR=/tmp/tmp.uFgy37Zjek ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8jmV7BpeLU ++ cat /tmp/tmp.uFgy37Zjek ++ rm /tmp/tmp.8jmV7BpeLU /tmp/tmp.uFgy37Zjek ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4At1RaNJ8k +++ mktemp ++ local LAST_ERR=/tmp/tmp.KFjONIcDAB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4At1RaNJ8k ++ cat /tmp/tmp.KFjONIcDAB ++ rm /tmp/tmp.4At1RaNJ8k /tmp/tmp.KFjONIcDAB ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness.......................................... + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MYwkhcDdWc +++ mktemp ++ local LAST_ERR=/tmp/tmp.PDo3EHC8zv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MYwkhcDdWc ++ cat /tmp/tmp.PDo3EHC8zv ++ rm /tmp/tmp.MYwkhcDdWc /tmp/tmp.PDo3EHC8zv ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vfSDIAKVdr +++ mktemp ++ local LAST_ERR=/tmp/tmp.du4Su7dXKC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vfSDIAKVdr ++ cat /tmp/tmp.du4Su7dXKC ++ rm /tmp/tmp.vfSDIAKVdr /tmp/tmp.du4Su7dXKC ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RgE5P7axhn +++ mktemp ++ local LAST_ERR=/tmp/tmp.b9B16OZR46 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RgE5P7axhn ++ cat /tmp/tmp.b9B16OZR46 ++ rm /tmp/tmp.RgE5P7axhn /tmp/tmp.b9B16OZR46 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + wait_for_running some-name-rs1 3 + local name=some-name-rs1 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs1-0 + local pod=some-name-rs1-0 + set +o xtrace waiting for pod/some-name-rs1-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs1-1 + local pod=some-name-rs1-1 + set +o xtrace waiting for pod/some-name-rs1-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tGjKuCzGDZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.BTtEjBkwPx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tGjKuCzGDZ ++ cat /tmp/tmp.BTtEjBkwPx ++ rm /tmp/tmp.tGjKuCzGDZ /tmp/tmp.BTtEjBkwPx ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs1-2 + local pod=some-name-rs1-2 + set +o xtrace waiting for pod/some-name-rs1-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uf0oFaCe5s +++ mktemp ++ local LAST_ERR=/tmp/tmp.5LFGQhBPPV ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uf0oFaCe5s ++ cat /tmp/tmp.5LFGQhBPPV ++ rm /tmp/tmp.uf0oFaCe5s /tmp/tmp.5LFGQhBPPV ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KGib5jA4dG +++ mktemp ++ local LAST_ERR=/tmp/tmp.cPKZ4KL4uf ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KGib5jA4dG ++ cat /tmp/tmp.cPKZ4KL4uf ++ rm /tmp/tmp.KGib5jA4dG /tmp/tmp.cPKZ4KL4uf ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness + wait_for_running some-name-rs2 3 + local name=some-name-rs2 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs2 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs2-0 + local pod=some-name-rs2-0 + set +o xtrace waiting for pod/some-name-rs2-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs2-1 + local pod=some-name-rs2-1 + set +o xtrace waiting for pod/some-name-rs2-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hpX6bxLa6e +++ mktemp ++ local LAST_ERR=/tmp/tmp.CpEpFPzhlb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hpX6bxLa6e ++ cat /tmp/tmp.CpEpFPzhlb ++ rm /tmp/tmp.hpX6bxLa6e /tmp/tmp.CpEpFPzhlb ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs2-2 + local pod=some-name-rs2-2 + set +o xtrace waiting for pod/some-name-rs2-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.502ied3LXC +++ mktemp ++ local LAST_ERR=/tmp/tmp.JzD59VHSaL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.502ied3LXC ++ cat /tmp/tmp.JzD59VHSaL ++ rm /tmp/tmp.502ied3LXC /tmp/tmp.JzD59VHSaL ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.I0swEaKcGn +++ mktemp ++ local LAST_ERR=/tmp/tmp.RN2NFEmiwZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.I0swEaKcGn ++ cat /tmp/tmp.RN2NFEmiwZ ++ rm /tmp/tmp.I0swEaKcGn /tmp/tmp.RN2NFEmiwZ ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness + sleep 10 + write_initial_data + desc 'create user myApp' + set +o xtrace ----------------------------------------------------------------------------------- create user myApp ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-352 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rB0sVEbTeQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.UbseprzNp3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rB0sVEbTeQ ++ cat /tmp/tmp.UbseprzNp3 ++ rm /tmp/tmp.rB0sVEbTeQ /tmp/tmp.UbseprzNp3 ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.eNGoOyuJVX ++ mktemp + local LAST_ERR=/tmp/tmp.GdC0QPF99G + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eNGoOyuJVX Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("a8e40ca0-ebc5-46cc-ba43-bf1ab47ea177") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.GdC0QPF99G + rm /tmp/tmp.eNGoOyuJVX /tmp/tmp.GdC0QPF99G + return 0 + sleep 2 + write_document + local cmp_postfix= + local sleep_value=0 + log 'write initial data, read from all' + set +o xtrace [2026-06-22T11:31:53+0000] write initial data, read from all + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.N3kmpkL0oM +++ mktemp ++ local LAST_ERR=/tmp/tmp.NBNPOLa0XO ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.N3kmpkL0oM ++ cat /tmp/tmp.NBNPOLa0XO ++ rm /tmp/tmp.N3kmpkL0oM /tmp/tmp.NBNPOLa0XO ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.Hgq5HZCuSt ++ mktemp + local LAST_ERR=/tmp/tmp.n3a1PrFpFC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Hgq5HZCuSt Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("bd7d7cda-fdcf-4122-a9d8-7c9c61bd9111") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.n3a1PrFpFC + rm /tmp/tmp.Hgq5HZCuSt /tmp/tmp.n3a1PrFpFC + return 0 + sleep 0 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-06-22T11:31:56+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FxsY1kuNgg +++ mktemp ++ local LAST_ERR=/tmp/tmp.akQZFevIxT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FxsY1kuNgg ++ cat /tmp/tmp.akQZFevIxT ++ rm /tmp/tmp.FxsY1kuNgg /tmp/tmp.akQZFevIxT ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.aXjtzHcQRm ++ mktemp + local LAST_ERR=/tmp/tmp.BVSFQ9qSHp + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aXjtzHcQRm + cat /tmp/tmp.BVSFQ9qSHp + rm /tmp/tmp.aXjtzHcQRm /tmp/tmp.BVSFQ9qSHp + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.JwNUXLpdvn/find + desc 'restore pitr type latest using backupSource' + set +o xtrace ----------------------------------------------------------------------------------- restore pitr type latest using backupSource ----------------------------------------------------------------------------------- + write_document -2nd + local cmp_postfix=-2nd + local sleep_value=0 + log 'write initial data, read from all' + set +o xtrace [2026-06-22T11:31:58+0000] write initial data, read from all + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uqpD4CIoUu +++ mktemp ++ local LAST_ERR=/tmp/tmp.XP1QpoBTjm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uqpD4CIoUu ++ cat /tmp/tmp.XP1QpoBTjm ++ rm /tmp/tmp.uqpD4CIoUu /tmp/tmp.XP1QpoBTjm ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.ZKI9W3skHU ++ mktemp + local LAST_ERR=/tmp/tmp.taOfdRE5OJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZKI9W3skHU Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("42daa7db-8347-4441-a8ad-2fd52052bdfd") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.taOfdRE5OJ + rm /tmp/tmp.ZKI9W3skHU /tmp/tmp.taOfdRE5OJ + return 0 + sleep 0 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 -2nd + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-06-22T11:32:01+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 mongodb '' '' 27017 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hUPtkIkrdC +++ mktemp ++ local LAST_ERR=/tmp/tmp.KLtOWjxod7 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hUPtkIkrdC ++ cat /tmp/tmp.KLtOWjxod7 ++ rm /tmp/tmp.hUPtkIkrdC /tmp/tmp.KLtOWjxod7 ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.YuVsDoFeam ++ mktemp + local LAST_ERR=/tmp/tmp.QK2CU2TCbi + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YuVsDoFeam + cat /tmp/tmp.QK2CU2TCbi + rm /tmp/tmp.YuVsDoFeam /tmp/tmp.QK2CU2TCbi + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/compare/find-2nd.json /tmp/tmp.JwNUXLpdvn/find-2nd + run_backup backup-minio 2 physical + local name=backup-minio + local idx=2 + local type=physical + desc 'run backup backup-minio-2' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-2 ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml + /usr/sbin/sed -e 's/name:/name: backup-minio-2/' + /usr/sbin/sed -e 's/type:/type: physical/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.XNcDwiStPK ++ mktemp + local LAST_ERR=/tmp/tmp.gkt1iX8MEj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XNcDwiStPK perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created + cat /tmp/tmp.gkt1iX8MEj + rm /tmp/tmp.XNcDwiStPK /tmp/tmp.gkt1iX8MEj + return 0 + wait_backup backup-minio-2 + local backup_name=backup-minio-2 + local target_state=ready + set +o xtrace waiting for backup-minio-2 to reach ready state..........OK + compare_latest_restorable_time some-name-rs0 backup-minio-2 + local cluster=some-name-rs0 + local backup_name=backup-minio-2 + local latest_restorable_time + local backup_time ++ get_latest_restorable_time some-name-rs0 ++ local cluster=some-name-rs0 ++ local first_timestamp ++ local second_timestamp ++ local retry=0 ++ [[ '' != '' ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.stpXfg13Y0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KRNU9oztJS +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.stpXfg13Y0 +++ cat /tmp/tmp.KRNU9oztJS +++ rm /tmp/tmp.stpXfg13Y0 /tmp/tmp.KRNU9oztJS +++ return 0 ++ first_timestamp=1782126834 ++ sleep 5 ++ [[ 1782126834 != '' ]] ++ [[ 1782126834 != null ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kX98CmpyzQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RaWHlTeMVX +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.kX98CmpyzQ +++ cat /tmp/tmp.RaWHlTeMVX +++ rm /tmp/tmp.kX98CmpyzQ /tmp/tmp.RaWHlTeMVX +++ return 0 ++ second_timestamp=1782126834 ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 1782126834 != '' ]] ++ [[ 1782126834 != null ]] ++ [[ 1782126834 == 1782126834 ]] ++ /usr/sbin/date -u -d @1782126834 +%Y-%m-%dT%H:%M:%SZ + latest_restorable_time=2026-06-22T11:13:54Z ++ get_latest_restorable_time_from_backup_object backup-minio-2 ++ local backup_name=backup-minio-2 ++ local latestRestorableTime ++ local retry=0 ++ [[ '' != '' ]] ++ sleep 5 +++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MGGxP5pUH2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lFHqVyt9j7 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.MGGxP5pUH2 +++ cat /tmp/tmp.lFHqVyt9j7 +++ rm /tmp/tmp.MGGxP5pUH2 /tmp/tmp.lFHqVyt9j7 +++ return 0 ++ latestRestorableTime=2026-06-22T11:13:54Z ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 2026-06-22T11:13:54Z != '' ]] ++ [[ 2026-06-22T11:13:54Z != null ]] ++ echo 2026-06-22T11:13:54Z + backup_time=2026-06-22T11:13:54Z + [[ 2026-06-22T11:13:54Z != 2026\-06\-22T11\:13\:54Z ]] + check_recovery backup-minio-2 latest '' -3rd some-name backupSource + local backup_name=backup-minio-2 + local restore_type=latest + local restore_date= + local cmp_postfix=-3rd + local cluster_name=some-name + local backupSource=backupSource ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uwLmI3hSyN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oRWKxJkTnW +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.uwLmI3hSyN +++ cat /tmp/tmp.oRWKxJkTnW +++ rm /tmp/tmp.uwLmI3hSyN /tmp/tmp.oRWKxJkTnW +++ return 0 ++ echo 1782126834 + local latest_ts=1782126834 + desc 'write more data before restore by latest' + set +o xtrace ----------------------------------------------------------------------------------- write more data before restore by latest ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Nb00LnEDsY +++ mktemp ++ local LAST_ERR=/tmp/tmp.RbtmWBBh0S ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Nb00LnEDsY ++ cat /tmp/tmp.RbtmWBBh0S ++ rm /tmp/tmp.Nb00LnEDsY /tmp/tmp.RbtmWBBh0S ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.OpH1cB7TCW ++ mktemp + local LAST_ERR=/tmp/tmp.jZCkB9oI56 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OpH1cB7TCW Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("ba6ae66f-9d7f-4938-964d-10644293b4a1") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.jZCkB9oI56 + rm /tmp/tmp.OpH1cB7TCW /tmp/tmp.jZCkB9oI56 + return 0 + [[ -n '' ]] + desc 'Restoring to latest' + set +o xtrace ----------------------------------------------------------------------------------- Restoring to latest ----------------------------------------------------------------------------------- ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bH4DXHm1eu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7ypHArTlRY +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.bH4DXHm1eu +++ cat /tmp/tmp.7ypHArTlRY +++ rm /tmp/tmp.bH4DXHm1eu /tmp/tmp.7ypHArTlRY +++ return 0 ++ echo 1782126834 + local current_ts=1782126834 + retries=0 + [[ 1782126834 -gt 1782126834 ]] + [[ 0 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.E7ghPHFyg8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XH5yTbrdkt +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.E7ghPHFyg8 +++ cat /tmp/tmp.XH5yTbrdkt +++ rm /tmp/tmp.E7ghPHFyg8 /tmp/tmp.XH5yTbrdkt +++ return 0 ++ echo 1782126834 + latest_ts=1782126834 + retries=1 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 + echo 'Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54)' Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54) + sleep 10 + [[ 1782126834 -gt 1782126834 ]] + [[ 1 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0FlPrz0nCG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.URgHHi7bDF +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.0FlPrz0nCG +++ cat /tmp/tmp.URgHHi7bDF +++ rm /tmp/tmp.0FlPrz0nCG /tmp/tmp.URgHHi7bDF +++ return 0 ++ echo 1782126834 + latest_ts=1782126834 + retries=2 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 + echo 'Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54)' Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54) + sleep 10 + [[ 1782126834 -gt 1782126834 ]] + [[ 2 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ijPxE8VyW1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bEo3KULhQZ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ijPxE8VyW1 +++ cat /tmp/tmp.bEo3KULhQZ +++ rm /tmp/tmp.ijPxE8VyW1 /tmp/tmp.bEo3KULhQZ +++ return 0 ++ echo 1782126834 + latest_ts=1782126834 + retries=3 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 + echo 'Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54)' Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54) + sleep 10 + [[ 1782126834 -gt 1782126834 ]] + [[ 3 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BGqSKvJQPF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TNr2ugJBD2 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.BGqSKvJQPF +++ cat /tmp/tmp.TNr2ugJBD2 +++ rm /tmp/tmp.BGqSKvJQPF /tmp/tmp.TNr2ugJBD2 +++ return 0 ++ echo 1782126834 + latest_ts=1782126834 + retries=4 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 + echo 'Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54)' Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54) + sleep 10 + [[ 1782126834 -gt 1782126834 ]] + [[ 4 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.wnMaH8rquz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tTmmV3hTPL +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.wnMaH8rquz +++ cat /tmp/tmp.tTmmV3hTPL +++ rm /tmp/tmp.wnMaH8rquz /tmp/tmp.tTmmV3hTPL +++ return 0 ++ echo 1782126834 + latest_ts=1782126834 + retries=5 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 + echo 'Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54)' Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54) + sleep 10 + [[ 1782126834 -gt 1782126834 ]] + [[ 5 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bssWDoHxWa ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VgORAVT8fX +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.bssWDoHxWa +++ cat /tmp/tmp.VgORAVT8fX +++ rm /tmp/tmp.bssWDoHxWa /tmp/tmp.VgORAVT8fX +++ return 0 ++ echo 1782126834 + latest_ts=1782126834 + retries=6 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 + echo 'Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54)' Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54) + sleep 10 + [[ 1782126834 -gt 1782126834 ]] + [[ 6 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.K6MYEn3iZI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WrCoHsUoD8 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.K6MYEn3iZI +++ cat /tmp/tmp.WrCoHsUoD8 +++ rm /tmp/tmp.K6MYEn3iZI /tmp/tmp.WrCoHsUoD8 +++ return 0 ++ echo 1782126834 + latest_ts=1782126834 + retries=7 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 + echo 'Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54)' Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54) + sleep 10 + [[ 1782126834 -gt 1782126834 ]] + [[ 7 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UJEepVJzq8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wkxmT6h5ia +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.UJEepVJzq8 +++ cat /tmp/tmp.wkxmT6h5ia +++ rm /tmp/tmp.UJEepVJzq8 /tmp/tmp.wkxmT6h5ia +++ return 0 ++ echo 1782126834 + latest_ts=1782126834 + retries=8 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 + echo 'Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54)' Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54) + sleep 10 + [[ 1782126834 -gt 1782126834 ]] + [[ 8 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tNJaX9JVpa ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6wg8ksqnIq +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.tNJaX9JVpa +++ cat /tmp/tmp.6wg8ksqnIq +++ rm /tmp/tmp.tNJaX9JVpa /tmp/tmp.6wg8ksqnIq +++ return 0 ++ echo 1782126834 + latest_ts=1782126834 + retries=9 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 + echo 'Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54)' Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54) + sleep 10 + [[ 1782126834 -gt 1782126834 ]] + [[ 9 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.P5lztS1P0m ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7XTJoI4YCk +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.P5lztS1P0m +++ cat /tmp/tmp.7XTJoI4YCk +++ rm /tmp/tmp.P5lztS1P0m /tmp/tmp.7XTJoI4YCk +++ return 0 ++ echo 1782126834 + latest_ts=1782126834 + retries=10 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 + echo 'Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54)' Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54) + sleep 10 + [[ 1782126834 -gt 1782126834 ]] + [[ 10 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hHpNoqOosy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.mY29vWGGmw +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.hHpNoqOosy +++ cat /tmp/tmp.mY29vWGGmw +++ rm /tmp/tmp.hHpNoqOosy /tmp/tmp.mY29vWGGmw +++ return 0 ++ echo 1782126834 + latest_ts=1782126834 + retries=11 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 + echo 'Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54)' Waiting for last oplog chunk (2026-06-22 11:13:54) to be 120 seconds older than starting chunk (2026-06-22 11:13:54) + sleep 10 + [[ 1782126834 -gt 1782126834 ]] + [[ 11 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QTNBqBqtTh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vfR4qpRtSV +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.QTNBqBqtTh +++ cat /tmp/tmp.vfR4qpRtSV +++ rm /tmp/tmp.QTNBqBqtTh /tmp/tmp.vfR4qpRtSV +++ return 0 ++ echo 1782128082 + latest_ts=1782128082 + retries=12 ++ format_date 1782128082 ++ local timestamp=1782128082 +++ TZ=UTC +++ /usr/sbin/date -d@1782128082 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:34:42 ++ format_date 1782126834 ++ local timestamp=1782126834 +++ TZ=UTC +++ /usr/sbin/date -d@1782126834 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-06-22 11:13:54 + echo 'Waiting for last oplog chunk (2026-06-22 11:34:42) to be 120 seconds older than starting chunk (2026-06-22 11:13:54)' Waiting for last oplog chunk (2026-06-22 11:34:42) to be 120 seconds older than starting chunk (2026-06-22 11:13:54) + sleep 10 + [[ 1782128082 -gt 1782126834 ]] + '[' -z backupSource ']' + desc 'check restore by latest backupSource' + set +o xtrace ----------------------------------------------------------------------------------- check restore by latest backupSource ----------------------------------------------------------------------------------- ++ get_backup_dest backup-minio-2 ++ local backup_name=backup-minio-2 ++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' +++ mktemp ++ sed 's|s3://||' ++ sed 's|azure://||' ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.NCSsqDOWOy +++ mktemp ++ local LAST_ERR=/tmp/tmp.XxakIdUvFE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NCSsqDOWOy ++ cat /tmp/tmp.XxakIdUvFE ++ rm /tmp/tmp.NCSsqDOWOy /tmp/tmp.XxakIdUvFE ++ return 0 + backup_dest=operator-testing/2026-06-22T11:32:06Z + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-2/' + /usr/sbin/sed -e /backupName/d + /usr/sbin/sed -e 's/pitrType:/type: latest/' + '[' -z '' ']' + /usr/sbin/sed -e /date:/d + /usr/sbin/sed -e 's|DESTINATION|operator-testing/2026-06-22T11:32:06Z|' + '[' -n '' ']' + yq + kubectl_bin apply -f - ++ mktemp ++ get_bucket_name backup-minio-2 ++ local backup_name=backup-minio-2 ++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.s3.bucket}' + local LAST_OUT=/tmp/tmp.Budf964J09 +++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.4VqaL2qoS6 + local exit_status=0 + local timeout=4 ++ local LAST_OUT=/tmp/tmp.Hg5JF2cvfr ++ seq 0 2 +++ mktemp + for i in $(seq 0 2) + set +e + kubectl apply -f - ++ local LAST_ERR=/tmp/tmp.DsOusabXNR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.s3.bucket}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Hg5JF2cvfr ++ cat /tmp/tmp.DsOusabXNR ++ rm /tmp/tmp.Hg5JF2cvfr /tmp/tmp.DsOusabXNR ++ return 0 + /usr/sbin/sed -e 's|BUCKET-NAME|operator-testing|' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Budf964J09 perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2 created + cat /tmp/tmp.4VqaL2qoS6 + rm /tmp/tmp.Budf964J09 /tmp/tmp.4VqaL2qoS6 + return 0 + wait_restore backup-minio-2 some-name requested 0 1200 + local backup_name=backup-minio-2 + local cluster_name=some-name + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=1200 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK Waiting psmdb-restore/restore-backup-minio-2 to reach state "requested" ........OK after 7 minutes + [[ 0 -eq 1 ]] + echo + wait_restore backup-minio-2 some-name ready 0 1600 + local backup_name=backup-minio-2 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=0 + local wait_time=1600 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK Waiting psmdb-restore/restore-backup-minio-2 to reach state "ready" ...OK after 2 minutes + [[ 0 -eq 1 ]] + echo + set -o xtrace + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BENPxOlfn3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.VmVa4nVCbi ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BENPxOlfn3 ++ cat /tmp/tmp.VmVa4nVCbi ++ rm /tmp/tmp.BENPxOlfn3 /tmp/tmp.VmVa4nVCbi ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.s1PpkhLAL9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Hkn6YtxWZY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.s1PpkhLAL9 ++ cat /tmp/tmp.Hkn6YtxWZY ++ rm /tmp/tmp.s1PpkhLAL9 /tmp/tmp.Hkn6YtxWZY ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K4aQIwt9Lt +++ mktemp ++ local LAST_ERR=/tmp/tmp.Rn7YCg3PZu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.K4aQIwt9Lt ++ cat /tmp/tmp.Rn7YCg3PZu ++ rm /tmp/tmp.K4aQIwt9Lt /tmp/tmp.Rn7YCg3PZu ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness............................................. + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iQGwkulzkY +++ mktemp ++ local LAST_ERR=/tmp/tmp.HMrLARWyp6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iQGwkulzkY ++ cat /tmp/tmp.HMrLARWyp6 ++ rm /tmp/tmp.iQGwkulzkY /tmp/tmp.HMrLARWyp6 ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oxan4SUyYc +++ mktemp ++ local LAST_ERR=/tmp/tmp.6FtKoCNquY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oxan4SUyYc ++ cat /tmp/tmp.6FtKoCNquY ++ rm /tmp/tmp.oxan4SUyYc /tmp/tmp.6FtKoCNquY ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ON8JMAVgsC +++ mktemp ++ local LAST_ERR=/tmp/tmp.90HZ1qLMdI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ON8JMAVgsC ++ cat /tmp/tmp.90HZ1qLMdI ++ rm /tmp/tmp.ON8JMAVgsC /tmp/tmp.90HZ1qLMdI ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WyAgP5I2o4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ffrz42y9uP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WyAgP5I2o4 ++ cat /tmp/tmp.Ffrz42y9uP ++ rm /tmp/tmp.WyAgP5I2o4 /tmp/tmp.Ffrz42y9uP ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EBOCj6FUOu +++ mktemp ++ local LAST_ERR=/tmp/tmp.e50mtOkQmi ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EBOCj6FUOu ++ cat /tmp/tmp.e50mtOkQmi ++ rm /tmp/tmp.EBOCj6FUOu /tmp/tmp.e50mtOkQmi ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KBp5XFetPs +++ mktemp ++ local LAST_ERR=/tmp/tmp.NmzlLFddaM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KBp5XFetPs ++ cat /tmp/tmp.NmzlLFddaM ++ rm /tmp/tmp.KBp5XFetPs /tmp/tmp.NmzlLFddaM ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness + sleep 10 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 -3rd + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local postfix=-3rd + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-06-22T11:47:59+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-352 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local mongo_flag= + local port=27017 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iij1EGJuUE +++ mktemp ++ local LAST_ERR=/tmp/tmp.RH1ezRlS2p ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iij1EGJuUE ++ cat /tmp/tmp.RH1ezRlS2p ++ rm /tmp/tmp.iij1EGJuUE /tmp/tmp.RH1ezRlS2p ++ return 0 + local client_container=psmdb-client-bb8b97679-82f4r + kubectl_bin exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.AVjub8IhfS ++ mktemp + local LAST_ERR=/tmp/tmp.YclR3lsFDg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-82f4r -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-352.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AVjub8IhfS + cat /tmp/tmp.YclR3lsFDg + rm /tmp/tmp.AVjub8IhfS /tmp/tmp.YclR3lsFDg + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/e2e-tests/pitr-physical-backup-source/compare/find-3rd.json /tmp/tmp.JwNUXLpdvn/find-3rd + desc 'disable pitr' + set +o xtrace ----------------------------------------------------------------------------------- disable pitr ----------------------------------------------------------------------------------- + kubectl patch psmdb some-name --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": false}}}}' perconaservermongodb.psmdb.percona.com/some-name patched + sleep 20 + desc 'delete all backups' + set +o xtrace ----------------------------------------------------------------------------------- delete all backups ----------------------------------------------------------------------------------- + kubectl_bin delete psmdb-backup --all ++ mktemp + local LAST_OUT=/tmp/tmp.6uz9mbAKt8 ++ mktemp + local LAST_ERR=/tmp/tmp.0gY7kgNBZm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb-backup --all + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6uz9mbAKt8 perconaservermongodbbackup.psmdb.percona.com "backup-minio-1" deleted from pitr-physical-backup-source-352 namespace perconaservermongodbbackup.psmdb.percona.com "backup-minio-2" deleted from pitr-physical-backup-source-352 namespace + cat /tmp/tmp.0gY7kgNBZm + rm /tmp/tmp.6uz9mbAKt8 /tmp/tmp.0gY7kgNBZm + return 0 + desc 'destroy cluster' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster ----------------------------------------------------------------------------------- + destroy pitr-physical-backup-source-352 + local namespace=pitr-physical-backup-source-352 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_backups + desc 'Delete psmdb-backup' + set +o xtrace ----------------------------------------------------------------------------------- Delete psmdb-backup ----------------------------------------------------------------------------------- ++ kubectl_bin get psmdb-backup --no-headers ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.l0OGTUJDDM +++ mktemp ++ local LAST_ERR=/tmp/tmp.XEeaAWMNxo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup --no-headers ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.l0OGTUJDDM ++ cat /tmp/tmp.XEeaAWMNxo No resources found in pitr-physical-backup-source-352 namespace. ++ rm /tmp/tmp.l0OGTUJDDM /tmp/tmp.XEeaAWMNxo ++ return 0 + '[' 0 '!=' 0 ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.6KoTJLNaCH ++ mktemp + local LAST_ERR=/tmp/tmp.E6dRU6PRCt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6KoTJLNaCH customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.E6dRU6PRCt + rm /tmp/tmp.6KoTJLNaCH /tmp/tmp.E6dRU6PRCt + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + grep -v NAMESPACE + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.GSjxI479He ++ mktemp + local LAST_ERR=/tmp/tmp.wzrcS9HFmT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GSjxI479He + cat /tmp/tmp.wzrcS9HFmT + rm /tmp/tmp.GSjxI479He /tmp/tmp.wzrcS9HFmT + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.V2nldb9ULS ++ mktemp + local LAST_ERR=/tmp/tmp.CpGsL5aXXU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.V2nldb9ULS + cat /tmp/tmp.CpGsL5aXXU + rm /tmp/tmp.V2nldb9ULS /tmp/tmp.CpGsL5aXXU + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.ZMW64y9HYp ++ mktemp + local LAST_ERR=/tmp/tmp.ZNapO8h7X4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZMW64y9HYp + cat /tmp/tmp.ZNapO8h7X4 + rm /tmp/tmp.ZMW64y9HYp /tmp/tmp.ZNapO8h7X4 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.wPQQAvJffH ++ mktemp + local LAST_ERR=/tmp/tmp.5RGjamGN1a + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wPQQAvJffH + cat /tmp/tmp.5RGjamGN1a + rm /tmp/tmp.wPQQAvJffH /tmp/tmp.5RGjamGN1a + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.pRHzO3nBt5 ++ mktemp + local LAST_ERR=/tmp/tmp.k0VeNxxg07 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2315/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pRHzO3nBt5 clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.k0VeNxxg07 + rm /tmp/tmp.pRHzO3nBt5 /tmp/tmp.k0VeNxxg07 + return 0 + destroy_cert_manager + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.xizOlRwdzj ++ mktemp + local LAST_ERR=/tmp/tmp.rivdc0iLHR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.xizOlRwdzj + cat /tmp/tmp.rivdc0iLHR Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.xizOlRwdzj + cat /tmp/tmp.rivdc0iLHR Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 4 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.xizOlRwdzj + cat /tmp/tmp.rivdc0iLHR Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 8 + cat /tmp/tmp.xizOlRwdzj + cat /tmp/tmp.rivdc0iLHR Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + rm /tmp/tmp.xizOlRwdzj /tmp/tmp.rivdc0iLHR + return 1 + true + '[' -n '' ']' + '[' -n psmdb-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace pitr-physical-backup-source-352 + rm -rf /tmp/tmp.JwNUXLpdvn + kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator ++ mktemp ++ mktemp + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.DT9I2CR68b ++ mktemp + local LAST_OUT=/tmp/tmp.dlI1RqF7CW ++ mktemp + local LAST_ERR=/tmp/tmp.9ngsleyYCj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace pitr-physical-backup-source-352 + local LAST_ERR=/tmp/tmp.hRoLJ5QqzX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace psmdb-operator