Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/logs/pitr.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 + main + create_infra pitr-24754 + local ns=pitr-24754 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.TeHTiuJfox ++ mktemp + local LAST_ERR=/tmp/tmp.xlXEoFcfIA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TeHTiuJfox customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.xlXEoFcfIA + rm /tmp/tmp.TeHTiuJfox /tmp/tmp.xlXEoFcfIA + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.9Op6biyMHM ++ mktemp + local LAST_ERR=/tmp/tmp.dZ202guItw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9Op6biyMHM + cat /tmp/tmp.dZ202guItw + rm /tmp/tmp.9Op6biyMHM /tmp/tmp.dZ202guItw + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.Ioww7gqIyF ++ mktemp + local LAST_ERR=/tmp/tmp.MFBkNd0XHN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Ioww7gqIyF + cat /tmp/tmp.MFBkNd0XHN + rm /tmp/tmp.Ioww7gqIyF /tmp/tmp.MFBkNd0XHN + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.3ryssMZTlX ++ mktemp + local LAST_ERR=/tmp/tmp.4zirnZYU6i + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3ryssMZTlX + cat /tmp/tmp.4zirnZYU6i + rm /tmp/tmp.3ryssMZTlX /tmp/tmp.4zirnZYU6i + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.cXDlFZ92Nz ++ mktemp + local LAST_ERR=/tmp/tmp.GvGpcwKPII + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cXDlFZ92Nz clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.GvGpcwKPII + rm /tmp/tmp.cXDlFZ92Nz /tmp/tmp.GvGpcwKPII + return 0 + check_crd_for_deletion PR-2272-ae4e3cbc + local git_tag=PR-2272-ae4e3cbc ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2272-ae4e3cbc/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hDqC0YZP7b +++ mktemp ++ local LAST_ERR=/tmp/tmp.2PWktyp08E ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.hDqC0YZP7b ++ cat /tmp/tmp.2PWktyp08E Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.hDqC0YZP7b ++ cat /tmp/tmp.2PWktyp08E Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.hDqC0YZP7b ++ cat /tmp/tmp.2PWktyp08E Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.hDqC0YZP7b ++ cat /tmp/tmp.2PWktyp08E Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.hDqC0YZP7b /tmp/tmp.2PWktyp08E ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ awk '{print $1}' ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrole ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found + kubectl_bin get ns ++ mktemp + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + local LAST_OUT=/tmp/tmp.PRntWJnQah ++ mktemp + xargs kubectl delete ns + local LAST_ERR=/tmp/tmp.mHERtlcimH + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ mktemp + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + local LAST_OUT=/tmp/tmp.3Qqe6LitmC ++ mktemp + local LAST_ERR=/tmp/tmp.FM7pFhCNKT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3Qqe6LitmC + cat /tmp/tmp.FM7pFhCNKT + rm /tmp/tmp.3Qqe6LitmC /tmp/tmp.FM7pFhCNKT + return 0 namespace "pitr-23889" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PRntWJnQah namespace "psmdb-operator" deleted + cat /tmp/tmp.mHERtlcimH + rm /tmp/tmp.PRntWJnQah /tmp/tmp.mHERtlcimH + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.JcN4DMk6ue ++ mktemp + local LAST_ERR=/tmp/tmp.QniE27gl8l + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JcN4DMk6ue + cat /tmp/tmp.QniE27gl8l + rm /tmp/tmp.JcN4DMk6ue /tmp/tmp.QniE27gl8l + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ljMcq1gDkm ++ mktemp + local LAST_ERR=/tmp/tmp.rzhTMrNxx9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ljMcq1gDkm namespace/psmdb-operator created + cat /tmp/tmp.rzhTMrNxx9 + rm /tmp/tmp.ljMcq1gDkm /tmp/tmp.rzhTMrNxx9 + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.8YaIqrI5BC +++ mktemp ++ local LAST_ERR=/tmp/tmp.Mb4dQMqCWj ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8YaIqrI5BC ++ cat /tmp/tmp.Mb4dQMqCWj ++ rm /tmp/tmp.8YaIqrI5BC /tmp/tmp.Mb4dQMqCWj ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2272-ae4e3cbc-4-cluster3 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.1MHiYDcAxq ++ mktemp + local LAST_ERR=/tmp/tmp.jtd9VAmgPn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2272-ae4e3cbc-4-cluster3 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1MHiYDcAxq Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2272-ae4e3cbc-4-cluster3" modified. + cat /tmp/tmp.jtd9VAmgPn + rm /tmp/tmp.1MHiYDcAxq /tmp/tmp.jtd9VAmgPn + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2272-ae4e3cbc' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2272-ae4e3cbc ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.XrMhF7sUy6 ++ mktemp + local LAST_ERR=/tmp/tmp.Np6z09u5g5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XrMhF7sUy6 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.Np6z09u5g5 + rm /tmp/tmp.XrMhF7sUy6 /tmp/tmp.Np6z09u5g5 + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/deploy/cw-rbac.yaml + kubectl_bin apply -n psmdb-operator -f - + sed -e 's^namespace: .*^namespace: psmdb-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.6lgqV7vz3M ++ mktemp + local LAST_ERR=/tmp/tmp.Gjc7Ix9QiV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6lgqV7vz3M clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.Gjc7Ix9QiV + rm /tmp/tmp.6lgqV7vz3M /tmp/tmp.Gjc7Ix9QiV + return 0 + yq eval ' (.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2272-ae4e3cbc") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Zw525cIfTF ++ mktemp + local LAST_ERR=/tmp/tmp.axcEAhZfE3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Zw525cIfTF deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.axcEAhZfE3 + rm /tmp/tmp.Zw525cIfTF /tmp/tmp.axcEAhZfE3 + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.edKxGzlz8o +++ mktemp ++ local LAST_ERR=/tmp/tmp.d5DgY8ZYSS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.edKxGzlz8o ++ cat /tmp/tmp.d5DgY8ZYSS ++ rm /tmp/tmp.edKxGzlz8o /tmp/tmp.d5DgY8ZYSS ++ return 0 + wait_operator_pod percona-server-mongodb-operator-6455bb5fb4-bmf5j + local pod=percona-server-mongodb-operator-6455bb5fb4-bmf5j + set +o xtrace waiting for pod/percona-server-mongodb-operator-6455bb5fb4-bmf5j to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZPEXm4WgIr +++ mktemp ++ local LAST_ERR=/tmp/tmp.t2LQ131EP5 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZPEXm4WgIr ++ cat /tmp/tmp.t2LQ131EP5 ++ rm /tmp/tmp.ZPEXm4WgIr /tmp/tmp.t2LQ131EP5 ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-6455bb5fb4-bmf5j ++ mktemp + local LAST_OUT=/tmp/tmp.elxBc9R9wQ ++ mktemp + local LAST_ERR=/tmp/tmp.RTCtdbqC1C + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-6455bb5fb4-bmf5j + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.elxBc9R9wQ + cat /tmp/tmp.RTCtdbqC1C + rm /tmp/tmp.elxBc9R9wQ /tmp/tmp.RTCtdbqC1C + return 0 2026-03-26T09:31:14.213Z INFO setup Manager starting up {"gitCommit": "ae4e3cbc053c422311418c8c6083b24139fc7e69", "gitBranch": "PR-2272-ae4e3cbc", "buildTime": "", "goVersion": "go1.25.8", "os": "linux", "arch": "amd64"} + create_namespace pitr-24754 + local namespace=pitr-24754 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ awk '{print $1}' ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrole ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns ++ mktemp + '[' -n '' ']' + desc 'cleaned up old namespaces pitr-24754' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pitr-24754 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pitr-24754 --ignore-not-found + xargs kubectl delete ns ++ mktemp + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + local LAST_OUT=/tmp/tmp.qYFJgAtFDF + local LAST_OUT=/tmp/tmp.JOJHHYFzM5 ++ mktemp + awk '{print$1}' ++ mktemp + local LAST_ERR=/tmp/tmp.KldmepSrKM + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.QGWgijrTyQ + local exit_status=0 + local timeout=4 + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace pitr-24754 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qYFJgAtFDF + cat /tmp/tmp.KldmepSrKM + rm /tmp/tmp.qYFJgAtFDF /tmp/tmp.KldmepSrKM + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JOJHHYFzM5 + cat /tmp/tmp.QGWgijrTyQ + rm /tmp/tmp.JOJHHYFzM5 /tmp/tmp.QGWgijrTyQ + return 0 + kubectl_bin wait --for=delete namespace pitr-24754 ++ mktemp + local LAST_OUT=/tmp/tmp.3IILl8KgcG ++ mktemp + local LAST_ERR=/tmp/tmp.V4zfGzGBTT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace pitr-24754 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3IILl8KgcG + cat /tmp/tmp.V4zfGzGBTT + rm /tmp/tmp.3IILl8KgcG /tmp/tmp.V4zfGzGBTT + return 0 + desc 'create namespace pitr-24754' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pitr-24754 ----------------------------------------------------------------------------------- + kubectl_bin create namespace pitr-24754 ++ mktemp + local LAST_OUT=/tmp/tmp.TVFZ6Jhzvd ++ mktemp + local LAST_ERR=/tmp/tmp.wcgJ0E4yNz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pitr-24754 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TVFZ6Jhzvd namespace/pitr-24754 created + cat /tmp/tmp.wcgJ0E4yNz + rm /tmp/tmp.TVFZ6Jhzvd /tmp/tmp.wcgJ0E4yNz + return 0 + set_kube_ctx pitr-24754 + local namespace=pitr-24754 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.cnkHGnoRwr +++ mktemp ++ local LAST_ERR=/tmp/tmp.T1AcSQVVUU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cnkHGnoRwr ++ cat /tmp/tmp.T1AcSQVVUU ++ rm /tmp/tmp.cnkHGnoRwr /tmp/tmp.T1AcSQVVUU ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2272-ae4e3cbc-4-cluster3 --namespace=pitr-24754 ++ mktemp + local LAST_OUT=/tmp/tmp.2342ntnl9F ++ mktemp + local LAST_ERR=/tmp/tmp.kvHRD6IBGI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2272-ae4e3cbc-4-cluster3 --namespace=pitr-24754 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2342ntnl9F Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2272-ae4e3cbc-4-cluster3" modified. + cat /tmp/tmp.kvHRD6IBGI + rm /tmp/tmp.2342ntnl9F /tmp/tmp.kvHRD6IBGI + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Thu Mar 26 09:31:52 2026 NAMESPACE: pitr-24754 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.pitr-24754.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace pitr-24754 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace pitr-24754 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-24754 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-24754 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YEuqOGbRYt +++ mktemp ++ local LAST_ERR=/tmp/tmp.ezKoYIFGBY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YEuqOGbRYt ++ cat /tmp/tmp.ezKoYIFGBY ++ rm /tmp/tmp.YEuqOGbRYt /tmp/tmp.ezKoYIFGBY ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-jdvr4 + wait_pod minio-service-6d5f646cdc-jdvr4 + local pod=minio-service-6d5f646cdc-jdvr4 + set +o xtrace waiting for pod/minio-service-6d5f646cdc-jdvr4 to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-24754.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.Q8tJvcxbeV ++ mktemp + local LAST_ERR=/tmp/tmp.in6KpD4rgd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Q8tJvcxbeV make_bucket: operator-testing pod "aws-cli" deleted from pitr-24754 namespace + cat /tmp/tmp.in6KpD4rgd + rm /tmp/tmp.Q8tJvcxbeV /tmp/tmp.in6KpD4rgd + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/conf/minio-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.DYXbaJ6eGX ++ mktemp + local LAST_ERR=/tmp/tmp.z4JMrO4e9K + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/conf/minio-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DYXbaJ6eGX secret/some-users created deployment.apps/psmdb-client created secret/minio-secret created + cat /tmp/tmp.z4JMrO4e9K + rm /tmp/tmp.DYXbaJ6eGX /tmp/tmp.z4JMrO4e9K + return 0 + cluster=some-name-rs0 + desc 'create first PSMDB cluster some-name-rs0' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster some-name-rs0 ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/conf/some-name-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/conf/some-name-rs0.yml + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/conf/some-name-rs0.yml + kubectl_bin apply -f - + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2272-ae4e3cbc"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/pitr-24754/g ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.vL0xmEsIqi ++ mktemp + local LAST_ERR=/tmp/tmp.aP8NyXRbCX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vL0xmEsIqi perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.aP8NyXRbCX + rm /tmp/tmp.vL0xmEsIqi /tmp/tmp.aP8NyXRbCX + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.......OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready......OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.doye6iaSSM +++ mktemp ++ local LAST_ERR=/tmp/tmp.rMKy2BWWjg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.doye6iaSSM ++ cat /tmp/tmp.rMKy2BWWjg ++ rm /tmp/tmp.doye6iaSSM /tmp/tmp.rMKy2BWWjg ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OWi371Wgrn +++ mktemp ++ local LAST_ERR=/tmp/tmp.FbJa95oZB2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OWi371Wgrn ++ cat /tmp/tmp.FbJa95oZB2 ++ rm /tmp/tmp.OWi371Wgrn /tmp/tmp.FbJa95oZB2 ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.piUDYEpRGM +++ mktemp ++ local LAST_ERR=/tmp/tmp.k9ra0KpEsu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.piUDYEpRGM ++ cat /tmp/tmp.k9ra0KpEsu ++ rm /tmp/tmp.piUDYEpRGM /tmp/tmp.k9ra0KpEsu ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness................. + desc 'check if statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/some-name-rs0 + local resource=statefulset/some-name-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/statefulset_some-name-rs0.yml + local new_result=/tmp/tmp.mG2dpeMM56/statefulset_some-name-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/statefulset_some-name-rs0-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("pitr-24754", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.hurWEy5RO6 ++ mktemp + local LAST_ERR=/tmp/tmp.rfWgf3zmSv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hurWEy5RO6 + cat /tmp/tmp.rfWgf3zmSv + rm /tmp/tmp.hurWEy5RO6 /tmp/tmp.rfWgf3zmSv + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.mG2dpeMM56/statefulset_some-name-rs0.yml + version_gt 1.22 ++ echo '1.32 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.mG2dpeMM56/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.mG2dpeMM56/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/statefulset_some-name-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/statefulset_some-name-rs0.yml /tmp/tmp.mG2dpeMM56/statefulset_some-name-rs0.yml + log 'compare_kubectl: statefulset/some-name-rs0 OK' + set +o xtrace [2026-03-26T09:34:27+0000] compare_kubectl: statefulset/some-name-rs0 OK + write_initial_data + desc 'create user myApp' + set +o xtrace ----------------------------------------------------------------------------------- create user myApp ----------------------------------------------------------------------------------- + run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.pitr-24754 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.pitr-24754 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.h6JoZuHbfS +++ mktemp ++ local LAST_ERR=/tmp/tmp.OzH2lqywQy ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.h6JoZuHbfS ++ cat /tmp/tmp.OzH2lqywQy ++ rm /tmp/tmp.h6JoZuHbfS /tmp/tmp.OzH2lqywQy ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.JVVVf5YIDV ++ mktemp + local LAST_ERR=/tmp/tmp.tmGeyOCIUm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JVVVf5YIDV Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("808af2a1-c814-423d-a6ba-4fd72b075e80") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.tmGeyOCIUm + rm /tmp/tmp.JVVVf5YIDV /tmp/tmp.tmGeyOCIUm + return 0 + sleep 2 + write_document + local cmp_postfix= + local cluster_name=some-name-rs0 + desc 'write initial data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write initial data, read from all ----------------------------------------------------------------------------------- + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pitr-24754 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.pitr-24754 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4ZXF7Q8kdx +++ mktemp ++ local LAST_ERR=/tmp/tmp.m86CcGYer1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4ZXF7Q8kdx ++ cat /tmp/tmp.m86CcGYer1 ++ rm /tmp/tmp.4ZXF7Q8kdx /tmp/tmp.m86CcGYer1 ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.RlrNF32cap ++ mktemp + local LAST_ERR=/tmp/tmp.Ln7TlL0mOv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RlrNF32cap Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("646c6971-1eb2-4d8e-98a3-a7ce63a147c4") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.Ln7TlL0mOv + rm /tmp/tmp.RlrNF32cap /tmp/tmp.Ln7TlL0mOv + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 '' + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:34:34+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v3AGmxQkbe +++ mktemp ++ local LAST_ERR=/tmp/tmp.CyoP3kikP7 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.v3AGmxQkbe ++ cat /tmp/tmp.CyoP3kikP7 ++ rm /tmp/tmp.v3AGmxQkbe /tmp/tmp.CyoP3kikP7 ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.YxE0lbYWZM ++ mktemp + local LAST_ERR=/tmp/tmp.ZbIQ3Nz7hG + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YxE0lbYWZM + cat /tmp/tmp.ZbIQ3Nz7hG + rm /tmp/tmp.YxE0lbYWZM /tmp/tmp.ZbIQ3Nz7hG + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find.json /tmp/tmp.mG2dpeMM56/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 '' + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:34:37+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aiQlQx4BmE +++ mktemp ++ local LAST_ERR=/tmp/tmp.X2k6jjekpA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aiQlQx4BmE ++ cat /tmp/tmp.X2k6jjekpA ++ rm /tmp/tmp.aiQlQx4BmE /tmp/tmp.X2k6jjekpA ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.SZHEYAQnNU ++ mktemp + local LAST_ERR=/tmp/tmp.no0BjAUqBM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SZHEYAQnNU + cat /tmp/tmp.no0BjAUqBM + rm /tmp/tmp.SZHEYAQnNU /tmp/tmp.no0BjAUqBM + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find.json /tmp/tmp.mG2dpeMM56/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 '' + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:34:40+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tctxv8MoKR +++ mktemp ++ local LAST_ERR=/tmp/tmp.LYOmrqgdc2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tctxv8MoKR ++ cat /tmp/tmp.LYOmrqgdc2 ++ rm /tmp/tmp.tctxv8MoKR /tmp/tmp.LYOmrqgdc2 ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.w8jXnnN57S ++ mktemp + local LAST_ERR=/tmp/tmp.Z3WmNearCj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.w8jXnnN57S + cat /tmp/tmp.Z3WmNearCj + rm /tmp/tmp.w8jXnnN57S /tmp/tmp.Z3WmNearCj + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find.json /tmp/tmp.mG2dpeMM56/find + wait_backup_agent some-name-rs0-0 + local agent_pod=some-name-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-0...2026-03-26T09:33:50.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-1 + local agent_pod=some-name-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-1...2026-03-26T09:34:20.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-2 + local agent_pod=some-name-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-2...2026-03-26T09:34:24.000+0000 I listening for the commands + backup_name_minio=backup-minio + run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-24754 + local 'command=use myApp\n db.test2.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.pitr-24754 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1yVgpxoz2A +++ mktemp ++ local LAST_ERR=/tmp/tmp.KawA6l9bTg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1yVgpxoz2A ++ cat /tmp/tmp.KawA6l9bTg ++ rm /tmp/tmp.1yVgpxoz2A /tmp/tmp.KawA6l9bTg ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.JtHcfpiz9h ++ mktemp + local LAST_ERR=/tmp/tmp.U8ns0lzyck + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JtHcfpiz9h Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("5916cd72-2299-4ce5-9ad7-723bba47903f") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.U8ns0lzyck + rm /tmp/tmp.JtHcfpiz9h /tmp/tmp.U8ns0lzyck + return 0 + run_backup backup-minio 0 + local name=backup-minio + local idx=0 + desc 'run backup backup-minio-0' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-0 ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/conf/backup-minio.yml + /usr/sbin/sed -e 's/name:/name: backup-minio-0/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.WjVOEci25W ++ mktemp + local LAST_ERR=/tmp/tmp.8dKOOy9wEu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WjVOEci25W perconaservermongodbbackup.psmdb.percona.com/backup-minio-0 created + cat /tmp/tmp.8dKOOy9wEu + rm /tmp/tmp.WjVOEci25W /tmp/tmp.8dKOOy9wEu + return 0 + wait_backup backup-minio-0 + local backup_name=backup-minio-0 + local target_state=ready + set +o xtrace waiting for backup-minio-0 to reach ready state.......OK + write_document -2nd + local cmp_postfix=-2nd + local cluster_name=some-name-rs0 + desc 'write initial data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write initial data, read from all ----------------------------------------------------------------------------------- + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pitr-24754 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.pitr-24754 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0lelzsaXEE +++ mktemp ++ local LAST_ERR=/tmp/tmp.sxNHEV8fVs ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0lelzsaXEE ++ cat /tmp/tmp.sxNHEV8fVs ++ rm /tmp/tmp.0lelzsaXEE /tmp/tmp.sxNHEV8fVs ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.lD1WvD4vRW ++ mktemp + local LAST_ERR=/tmp/tmp.WqQVoUbNFG + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lD1WvD4vRW Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("24a8e9e9-0ef0-432e-b40d-38c8688b09c9") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.WqQVoUbNFG + rm /tmp/tmp.lD1WvD4vRW /tmp/tmp.WqQVoUbNFG + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 -2nd + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:35:06+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 mongodb '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.68ux4dZylg +++ mktemp ++ local LAST_ERR=/tmp/tmp.oa8dUnRHos ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.68ux4dZylg ++ cat /tmp/tmp.oa8dUnRHos ++ rm /tmp/tmp.68ux4dZylg /tmp/tmp.oa8dUnRHos ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Ju2QTup7mZ ++ mktemp + local LAST_ERR=/tmp/tmp.l7Na9NevNN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Ju2QTup7mZ + cat /tmp/tmp.l7Na9NevNN + rm /tmp/tmp.Ju2QTup7mZ /tmp/tmp.l7Na9NevNN + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.mG2dpeMM56/find-2nd + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 -2nd + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:35:08+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.03XrPS8vXY +++ mktemp ++ local LAST_ERR=/tmp/tmp.NhnCvbMtGo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.03XrPS8vXY ++ cat /tmp/tmp.NhnCvbMtGo ++ rm /tmp/tmp.03XrPS8vXY /tmp/tmp.NhnCvbMtGo ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.T8vHguF0en ++ mktemp + local LAST_ERR=/tmp/tmp.TZNZrZFumB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.T8vHguF0en + cat /tmp/tmp.TZNZrZFumB + rm /tmp/tmp.T8vHguF0en /tmp/tmp.TZNZrZFumB + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.mG2dpeMM56/find-2nd + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 -2nd + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:35:11+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Az3iZzURKh +++ mktemp ++ local LAST_ERR=/tmp/tmp.SEsU1ThArg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Az3iZzURKh ++ cat /tmp/tmp.SEsU1ThArg ++ rm /tmp/tmp.Az3iZzURKh /tmp/tmp.SEsU1ThArg ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.TeEyPSNBnT ++ mktemp + local LAST_ERR=/tmp/tmp.k43pzpDj16 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TeEyPSNBnT + cat /tmp/tmp.k43pzpDj16 + rm /tmp/tmp.TeEyPSNBnT /tmp/tmp.k43pzpDj16 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.mG2dpeMM56/find-2nd + sleep 2 ++ run_mongo 'new Date().toISOString()' myApp:myPass@some-name-rs0.pitr-24754 mongodb '' --quiet ++ local 'command=new Date().toISOString()' ++ local uri=myApp:myPass@some-name-rs0.pitr-24754 ++ local driver=mongodb ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match' ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.pitr-24754 == *cfg* ]] ++ cut -c1-19 ++ tr T ' ' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6ZEqYUTrCn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JCLRNixTv0 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.6ZEqYUTrCn +++ cat /tmp/tmp.JCLRNixTv0 +++ rm /tmp/tmp.6ZEqYUTrCn /tmp/tmp.JCLRNixTv0 +++ return 0 ++ local client_container=psmdb-client-bb8b97679-zsz7m ++ kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NI6fCHYhQW +++ mktemp ++ local LAST_ERR=/tmp/tmp.bgpkxvaZc7 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NI6fCHYhQW ++ cat /tmp/tmp.bgpkxvaZc7 ++ rm /tmp/tmp.NI6fCHYhQW /tmp/tmp.bgpkxvaZc7 ++ return 0 + time_now='2026-03-26 09:35:18' + check_recovery backup-minio-0 date '2026-03-26 09:35:18' -2nd '' some-name test2 + local backup_name=backup-minio-0 + local restore_type=date + local 'restore_date=2026-03-26 09:35:18' + local cmp_postfix=-2nd + local backupSource= + local cluster_name=some-name + local selective_collection=test2 + local restore_name=restore-backup-minio-0 + local restore_file=restore.yml + local cluster + cluster=some-name-rs0 + desc 'write more data before restore by date' + set +o xtrace ----------------------------------------------------------------------------------- write more data before restore by date ----------------------------------------------------------------------------------- + sleep 60 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-24754 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.pitr-24754 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TFDU4roi3X +++ mktemp ++ local LAST_ERR=/tmp/tmp.sCH1FxYAj8 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TFDU4roi3X ++ cat /tmp/tmp.sCH1FxYAj8 ++ rm /tmp/tmp.TFDU4roi3X /tmp/tmp.sCH1FxYAj8 ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.SXnUw5ld4l ++ mktemp + local LAST_ERR=/tmp/tmp.xYBq6pyZaE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SXnUw5ld4l Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("6551e87b-0ea3-465e-becf-d61ee9646d62") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.xYBq6pyZaE + rm /tmp/tmp.SXnUw5ld4l /tmp/tmp.xYBq6pyZaE + return 0 + '[' -n test2 ']' + run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.pitr-24754 + local 'command=use myApp\n db.test2.drop()' + local uri=myApp:myPass@some-name-rs0.pitr-24754 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.52bOhUAhof +++ mktemp ++ local LAST_ERR=/tmp/tmp.42PU5Innun ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.52bOhUAhof ++ cat /tmp/tmp.42PU5Innun ++ rm /tmp/tmp.52bOhUAhof /tmp/tmp.42PU5Innun ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.oGiqF8Cz9d ++ mktemp + local LAST_ERR=/tmp/tmp.oZmXNXk62F + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oGiqF8Cz9d Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("a985fd9e-9c74-4f36-bf6d-5ccac855cbfd") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.oZmXNXk62F + rm /tmp/tmp.oGiqF8Cz9d /tmp/tmp.oZmXNXk62F + return 0 + desc 'waiting for chunks to be uploaded' + set +o xtrace ----------------------------------------------------------------------------------- waiting for chunks to be uploaded ----------------------------------------------------------------------------------- + sleep 120 + desc 'check restore by date' + set +o xtrace ----------------------------------------------------------------------------------- check restore by date ----------------------------------------------------------------------------------- + '[' -z '' ']' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-0/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-0/' + /usr/sbin/sed -e 's/clusterName:/clusterName: some-name/' + /usr/sbin/sed -e 's/pitrType:/type: date/' + kubectl_bin apply -f - + '[' -z '2026-03-26 09:35:18' ']' + /usr/sbin/sed -e 's/date:/date: 2026-03-26 09:35:18/' + /usr/sbin/sed -e /backupSource/,+2d + '[' -n test2 ']' + yq eval '.spec.selective = {"namespaces": ["myApp.test"], "withUsersAndRoles": true}' ++ mktemp + local LAST_OUT=/tmp/tmp.gH5nN3ilVX ++ mktemp + local LAST_ERR=/tmp/tmp.UizP9d975C + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gH5nN3ilVX perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-0 created + cat /tmp/tmp.UizP9d975C + rm /tmp/tmp.gH5nN3ilVX /tmp/tmp.UizP9d975C + return 0 + wait_restore backup-minio-0 some-name + local backup_name=backup-minio-0 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-0 object to be created.OK Waiting psmdb-restore/restore-backup-minio-0 to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tIzvLA8VUB +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ab9xNmJcEZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tIzvLA8VUB ++ cat /tmp/tmp.Ab9xNmJcEZ ++ rm /tmp/tmp.tIzvLA8VUB /tmp/tmp.Ab9xNmJcEZ ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + echo + set -o xtrace + '[' -n test2 ']' ++ collection_exists test2 ./e2e-tests/pitr/run: line 108: collection_exists: command not found + [[ '' == \t\r\u\e ]] + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 -2nd + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:38:51+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 mongodb '' '' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.s0Na5OyB2j +++ mktemp ++ local LAST_ERR=/tmp/tmp.blJwT3beIo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.s0Na5OyB2j ++ cat /tmp/tmp.blJwT3beIo ++ rm /tmp/tmp.s0Na5OyB2j /tmp/tmp.blJwT3beIo ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.4leiKr6E84 ++ mktemp + local LAST_ERR=/tmp/tmp.xa9SUKnpwM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4leiKr6E84 + cat /tmp/tmp.xa9SUKnpwM + rm /tmp/tmp.4leiKr6E84 /tmp/tmp.xa9SUKnpwM + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.mG2dpeMM56/find-2nd + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 -2nd + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:38:53+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.j772hUh0e6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.eoaX4Uqw5r ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.j772hUh0e6 ++ cat /tmp/tmp.eoaX4Uqw5r ++ rm /tmp/tmp.j772hUh0e6 /tmp/tmp.eoaX4Uqw5r ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.WfGhapBpaF ++ mktemp + local LAST_ERR=/tmp/tmp.0o1neNTrvc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WfGhapBpaF + cat /tmp/tmp.0o1neNTrvc + rm /tmp/tmp.WfGhapBpaF /tmp/tmp.0o1neNTrvc + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.mG2dpeMM56/find-2nd + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 -2nd + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:38:55+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 mongodb '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.N1spWg5nRo +++ mktemp ++ local LAST_ERR=/tmp/tmp.gmcHvBamsE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.N1spWg5nRo ++ cat /tmp/tmp.gmcHvBamsE ++ rm /tmp/tmp.N1spWg5nRo /tmp/tmp.gmcHvBamsE ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.gYXwRh4Ncc ++ mktemp + local LAST_ERR=/tmp/tmp.GBThLwHk3H + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gYXwRh4Ncc + cat /tmp/tmp.GBThLwHk3H + rm /tmp/tmp.gYXwRh4Ncc /tmp/tmp.GBThLwHk3H + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.mG2dpeMM56/find-2nd + run_backup backup-minio 1 + local name=backup-minio + local idx=1 + desc 'run backup backup-minio-1' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-1 ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/conf/backup-minio.yml + /usr/sbin/sed -e 's/name:/name: backup-minio-1/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.goQIUsBYGa ++ mktemp + local LAST_ERR=/tmp/tmp.RM17Qwl4TY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.goQIUsBYGa perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created + cat /tmp/tmp.RM17Qwl4TY + rm /tmp/tmp.goQIUsBYGa /tmp/tmp.RM17Qwl4TY + return 0 + wait_backup backup-minio-1 + local backup_name=backup-minio-1 + local target_state=ready + set +o xtrace waiting for backup-minio-1 to reach ready state.......OK + compare_latest_restorable_time some-name-rs0 backup-minio-1 + local cluster=some-name-rs0 + local backup_name=backup-minio-1 + local latest_restorable_time + local backup_time ++ get_latest_restorable_time some-name-rs0 ++ local cluster=some-name-rs0 ++ local first_timestamp ++ local second_timestamp ++ local retry=0 ++ [[ '' != '' ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Adx55lGU4m ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Fbf6htXOeN +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Adx55lGU4m +++ cat /tmp/tmp.Fbf6htXOeN +++ rm /tmp/tmp.Adx55lGU4m /tmp/tmp.Fbf6htXOeN +++ return 0 ++ first_timestamp=1774517909 ++ sleep 5 ++ [[ 1774517909 != '' ]] ++ [[ 1774517909 != \n\u\l\l ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.YQRaKYdKjZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GZHxmLpCr7 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.YQRaKYdKjZ +++ cat /tmp/tmp.GZHxmLpCr7 +++ rm /tmp/tmp.YQRaKYdKjZ /tmp/tmp.GZHxmLpCr7 +++ return 0 ++ second_timestamp=1774517909 ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 1774517909 != '' ]] ++ [[ 1774517909 != \n\u\l\l ]] ++ [[ 1774517909 == 1774517909 ]] ++ /usr/sbin/date -u -d @1774517909 +%Y-%m-%dT%H:%M:%SZ + latest_restorable_time=2026-03-26T09:38:29Z ++ get_latest_restorable_time_from_backup_object backup-minio-1 ++ local backup_name=backup-minio-1 ++ local latestRestorableTime ++ local retry=0 ++ [[ '' != '' ]] ++ sleep 5 +++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5xNeLwuWi8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GHYbJRWZK2 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.5xNeLwuWi8 +++ cat /tmp/tmp.GHYbJRWZK2 +++ rm /tmp/tmp.5xNeLwuWi8 /tmp/tmp.GHYbJRWZK2 +++ return 0 ++ latestRestorableTime=2026-03-26T09:38:29Z ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 2026-03-26T09:38:29Z != '' ]] ++ [[ 2026-03-26T09:38:29Z != \n\u\l\l ]] ++ echo 2026-03-26T09:38:29Z + backup_time=2026-03-26T09:38:29Z + [[ 2026-03-26T09:38:29Z != \2\0\2\6\-\0\3\-\2\6\T\0\9\:\3\8\:\2\9\Z ]] + check_recovery backup-minio-1 latest '' -3rd '' some-name + local backup_name=backup-minio-1 + local restore_type=latest + local restore_date= + local cmp_postfix=-3rd + local backupSource= + local cluster_name=some-name + local selective_collection= + local restore_name=restore-backup-minio-1 + local restore_file=restore.yml + local cluster + cluster=some-name-rs0 + desc 'write more data before restore by latest' + set +o xtrace ----------------------------------------------------------------------------------- write more data before restore by latest ----------------------------------------------------------------------------------- + sleep 60 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-24754 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.pitr-24754 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q05tjddPpq +++ mktemp ++ local LAST_ERR=/tmp/tmp.LLM7Pl0Jja ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.q05tjddPpq ++ cat /tmp/tmp.LLM7Pl0Jja ++ rm /tmp/tmp.q05tjddPpq /tmp/tmp.LLM7Pl0Jja ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.al4mr0wT1B ++ mktemp + local LAST_ERR=/tmp/tmp.GtwQoKKq9W + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.al4mr0wT1B Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("1cde87c8-1057-4cdd-8634-46e2f0713661") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.GtwQoKKq9W + rm /tmp/tmp.al4mr0wT1B /tmp/tmp.GtwQoKKq9W + return 0 + '[' -n '' ']' + desc 'waiting for chunks to be uploaded' + set +o xtrace ----------------------------------------------------------------------------------- waiting for chunks to be uploaded ----------------------------------------------------------------------------------- + sleep 120 + desc 'check restore by latest' + set +o xtrace ----------------------------------------------------------------------------------- check restore by latest ----------------------------------------------------------------------------------- + '[' -z '' ']' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-1/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-1/' + /usr/sbin/sed -e 's/pitrType:/type: latest/' + /usr/sbin/sed -e /backupSource/,+2d + /usr/sbin/sed -e 's/clusterName:/clusterName: some-name/' + '[' -z '' ']' + /usr/sbin/sed -e /date:/d + '[' -n '' ']' + yq + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.SIctee0YvU ++ mktemp + local LAST_ERR=/tmp/tmp.6nfUoOnXT5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SIctee0YvU perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created + cat /tmp/tmp.6nfUoOnXT5 + rm /tmp/tmp.SIctee0YvU /tmp/tmp.6nfUoOnXT5 + return 0 + wait_restore backup-minio-1 some-name + local backup_name=backup-minio-1 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK Waiting psmdb-restore/restore-backup-minio-1 to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xLuxEbtNJ9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.VNGJrJz3HI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xLuxEbtNJ9 ++ cat /tmp/tmp.VNGJrJz3HI ++ rm /tmp/tmp.xLuxEbtNJ9 /tmp/tmp.VNGJrJz3HI ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + echo + set -o xtrace + '[' -n '' ']' + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 -3rd + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 + local postfix=-3rd + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:42:57+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 mongodb '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QbgVz8bM1f +++ mktemp ++ local LAST_ERR=/tmp/tmp.ezlJsicgbJ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QbgVz8bM1f ++ cat /tmp/tmp.ezlJsicgbJ ++ rm /tmp/tmp.QbgVz8bM1f /tmp/tmp.ezlJsicgbJ ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.L6d9svIwcF ++ mktemp + local LAST_ERR=/tmp/tmp.UjrruZ8PIm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.L6d9svIwcF + cat /tmp/tmp.UjrruZ8PIm + rm /tmp/tmp.L6d9svIwcF /tmp/tmp.UjrruZ8PIm + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find-3rd.json /tmp/tmp.mG2dpeMM56/find-3rd + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 -3rd + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 + local postfix=-3rd + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:42:59+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZbNtAlBPNz +++ mktemp ++ local LAST_ERR=/tmp/tmp.aPtg3KhFwc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZbNtAlBPNz ++ cat /tmp/tmp.aPtg3KhFwc ++ rm /tmp/tmp.ZbNtAlBPNz /tmp/tmp.aPtg3KhFwc ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.a6bxcce7o3 ++ mktemp + local LAST_ERR=/tmp/tmp.1m2pwoBBwR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.a6bxcce7o3 + cat /tmp/tmp.1m2pwoBBwR + rm /tmp/tmp.a6bxcce7o3 /tmp/tmp.1m2pwoBBwR + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find-3rd.json /tmp/tmp.mG2dpeMM56/find-3rd + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 -3rd + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 + local postfix=-3rd + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:43:02+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.E8lDifmjr2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.pbOwkYVOFx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.E8lDifmjr2 ++ cat /tmp/tmp.pbOwkYVOFx ++ rm /tmp/tmp.E8lDifmjr2 /tmp/tmp.pbOwkYVOFx ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.kuFrBuOy3c ++ mktemp + local LAST_ERR=/tmp/tmp.gtKImDxm6R + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kuFrBuOy3c + cat /tmp/tmp.gtKImDxm6R + rm /tmp/tmp.kuFrBuOy3c /tmp/tmp.gtKImDxm6R + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find-3rd.json /tmp/tmp.mG2dpeMM56/find-3rd + desc 'restore pitr using backupSource' + set +o xtrace ----------------------------------------------------------------------------------- restore pitr using backupSource ----------------------------------------------------------------------------------- + reset_collection + desc 'reset data' + set +o xtrace ----------------------------------------------------------------------------------- reset data ----------------------------------------------------------------------------------- + run_mongo 'use myApp\n db.test.remove({})' myApp:myPass@some-name-rs0.pitr-24754 + local 'command=use myApp\n db.test.remove({})' + local uri=myApp:myPass@some-name-rs0.pitr-24754 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gotlOGj9hx +++ mktemp ++ local LAST_ERR=/tmp/tmp.N6HXAHR3RQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gotlOGj9hx ++ cat /tmp/tmp.N6HXAHR3RQ ++ rm /tmp/tmp.gotlOGj9hx /tmp/tmp.N6HXAHR3RQ ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.kOvTJ7B9tD ++ mktemp + local LAST_ERR=/tmp/tmp.kMt3XDXWgH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kOvTJ7B9tD Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("d017d8ea-7bd2-4eee-91ee-e2a5162ed8d9") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nRemoved" : 3 }) bye + cat /tmp/tmp.kMt3XDXWgH + rm /tmp/tmp.kOvTJ7B9tD /tmp/tmp.kMt3XDXWgH + return 0 + sleep 2 + write_document + local cmp_postfix= + local cluster_name=some-name-rs0 + desc 'write initial data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write initial data, read from all ----------------------------------------------------------------------------------- + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pitr-24754 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.pitr-24754 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PSFQZ42tiY +++ mktemp ++ local LAST_ERR=/tmp/tmp.FaVkywSJ5Y ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PSFQZ42tiY ++ cat /tmp/tmp.FaVkywSJ5Y ++ rm /tmp/tmp.PSFQZ42tiY /tmp/tmp.FaVkywSJ5Y ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.zVlIM3XR6T ++ mktemp + local LAST_ERR=/tmp/tmp.czmVKuSNFZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zVlIM3XR6T Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("345fff15-d2ae-47c6-985f-ba81e01e1766") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.czmVKuSNFZ + rm /tmp/tmp.zVlIM3XR6T /tmp/tmp.czmVKuSNFZ + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 '' + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:43:11+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LWf3LsePjL +++ mktemp ++ local LAST_ERR=/tmp/tmp.EXCgYaqZ54 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LWf3LsePjL ++ cat /tmp/tmp.EXCgYaqZ54 ++ rm /tmp/tmp.LWf3LsePjL /tmp/tmp.EXCgYaqZ54 ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.tPlVV2eXXJ ++ mktemp + local LAST_ERR=/tmp/tmp.Y2O30JSKLy + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tPlVV2eXXJ + cat /tmp/tmp.Y2O30JSKLy + rm /tmp/tmp.tPlVV2eXXJ /tmp/tmp.Y2O30JSKLy + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find.json /tmp/tmp.mG2dpeMM56/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 '' + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:43:13+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6gZe7ktICF +++ mktemp ++ local LAST_ERR=/tmp/tmp.k3h08D6R9D ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6gZe7ktICF ++ cat /tmp/tmp.k3h08D6R9D ++ rm /tmp/tmp.6gZe7ktICF /tmp/tmp.k3h08D6R9D ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.kX3VQVB2GP ++ mktemp + local LAST_ERR=/tmp/tmp.JaLYnSmAN0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kX3VQVB2GP + cat /tmp/tmp.JaLYnSmAN0 + rm /tmp/tmp.kX3VQVB2GP /tmp/tmp.JaLYnSmAN0 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find.json /tmp/tmp.mG2dpeMM56/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 '' + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:43:15+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BiPvjuvUXI +++ mktemp ++ local LAST_ERR=/tmp/tmp.sxKzfbt9Oy ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BiPvjuvUXI ++ cat /tmp/tmp.sxKzfbt9Oy ++ rm /tmp/tmp.BiPvjuvUXI /tmp/tmp.sxKzfbt9Oy ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.cQ7hd8uAIY ++ mktemp + local LAST_ERR=/tmp/tmp.wvblLdbmWD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cQ7hd8uAIY + cat /tmp/tmp.wvblLdbmWD + rm /tmp/tmp.cQ7hd8uAIY /tmp/tmp.wvblLdbmWD + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find.json /tmp/tmp.mG2dpeMM56/find + run_backup backup-minio 2 + local name=backup-minio + local idx=2 + desc 'run backup backup-minio-2' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-2 ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/conf/backup-minio.yml + /usr/sbin/sed -e 's/name:/name: backup-minio-2/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.nLJmGPWjfG ++ mktemp + local LAST_ERR=/tmp/tmp.voyXgRx66A + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nLJmGPWjfG perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created + cat /tmp/tmp.voyXgRx66A + rm /tmp/tmp.nLJmGPWjfG /tmp/tmp.voyXgRx66A + return 0 + wait_backup backup-minio-2 + local backup_name=backup-minio-2 + local target_state=ready + set +o xtrace waiting for backup-minio-2 to reach ready state.......OK + compare_latest_restorable_time some-name-rs0 backup-minio-2 + local cluster=some-name-rs0 + local backup_name=backup-minio-2 + local latest_restorable_time + local backup_time ++ get_latest_restorable_time some-name-rs0 ++ local cluster=some-name-rs0 ++ local first_timestamp ++ local second_timestamp ++ local retry=0 ++ [[ '' != '' ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.D2F2X02OEZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IBRh26aGdo +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.D2F2X02OEZ +++ cat /tmp/tmp.IBRh26aGdo +++ rm /tmp/tmp.D2F2X02OEZ /tmp/tmp.IBRh26aGdo +++ return 0 ++ first_timestamp=1774518155 ++ sleep 5 ++ [[ 1774518155 != '' ]] ++ [[ 1774518155 != \n\u\l\l ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0xDk81qcyI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dQ1OHAAY4n +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.0xDk81qcyI +++ cat /tmp/tmp.dQ1OHAAY4n +++ rm /tmp/tmp.0xDk81qcyI /tmp/tmp.dQ1OHAAY4n +++ return 0 ++ second_timestamp=1774518155 ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 1774518155 != '' ]] ++ [[ 1774518155 != \n\u\l\l ]] ++ [[ 1774518155 == 1774518155 ]] ++ /usr/sbin/date -u -d @1774518155 +%Y-%m-%dT%H:%M:%SZ + latest_restorable_time=2026-03-26T09:42:35Z ++ get_latest_restorable_time_from_backup_object backup-minio-2 ++ local backup_name=backup-minio-2 ++ local latestRestorableTime ++ local retry=0 ++ [[ '' != '' ]] ++ sleep 5 +++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vWeqNi32yy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vdF3J4m60V +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.vWeqNi32yy +++ cat /tmp/tmp.vdF3J4m60V +++ rm /tmp/tmp.vWeqNi32yy /tmp/tmp.vdF3J4m60V +++ return 0 ++ latestRestorableTime=2026-03-26T09:42:35Z ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 2026-03-26T09:42:35Z != '' ]] ++ [[ 2026-03-26T09:42:35Z != \n\u\l\l ]] ++ echo 2026-03-26T09:42:35Z + backup_time=2026-03-26T09:42:35Z + [[ 2026-03-26T09:42:35Z != \2\0\2\6\-\0\3\-\2\6\T\0\9\:\4\2\:\3\5\Z ]] ++ run_mongo 'new Date().toISOString()' myApp:myPass@some-name-rs0.pitr-24754 mongodb '' --quiet ++ local 'command=new Date().toISOString()' ++ local uri=myApp:myPass@some-name-rs0.pitr-24754 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.pitr-24754 == *cfg* ]] ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match' ++ cut -c1-19 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ tr T ' ' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EjDtKfGUAE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ATLQcDvWKi +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.EjDtKfGUAE +++ cat /tmp/tmp.ATLQcDvWKi +++ rm /tmp/tmp.EjDtKfGUAE /tmp/tmp.ATLQcDvWKi +++ return 0 ++ local client_container=psmdb-client-bb8b97679-zsz7m ++ kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fzYHXaP11Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.dr1NAmiEaZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fzYHXaP11Q ++ cat /tmp/tmp.dr1NAmiEaZ ++ rm /tmp/tmp.fzYHXaP11Q /tmp/tmp.dr1NAmiEaZ ++ return 0 + time_now='2026-03-26 09:43:48' + check_recovery backup-minio-2 date '2026-03-26 09:43:48' '' backupSource some-name + local backup_name=backup-minio-2 + local restore_type=date + local 'restore_date=2026-03-26 09:43:48' + local cmp_postfix= + local backupSource=backupSource + local cluster_name=some-name + local selective_collection= + local restore_name=restore-backup-minio-2 + local restore_file=restore.yml + local cluster + cluster=some-name-rs0 + desc 'write more data before restore by date' + set +o xtrace ----------------------------------------------------------------------------------- write more data before restore by date ----------------------------------------------------------------------------------- + sleep 60 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-24754 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.pitr-24754 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.anqN6BOhIX +++ mktemp ++ local LAST_ERR=/tmp/tmp.ABKIbKVHN2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.anqN6BOhIX ++ cat /tmp/tmp.ABKIbKVHN2 ++ rm /tmp/tmp.anqN6BOhIX /tmp/tmp.ABKIbKVHN2 ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.0lZYvDxtKt ++ mktemp + local LAST_ERR=/tmp/tmp.art4TMGD6J + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0lZYvDxtKt Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("c2f30057-eacd-44e8-a351-b9831698ea21") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.art4TMGD6J + rm /tmp/tmp.0lZYvDxtKt /tmp/tmp.art4TMGD6J + return 0 + '[' -n '' ']' + desc 'waiting for chunks to be uploaded' + set +o xtrace ----------------------------------------------------------------------------------- waiting for chunks to be uploaded ----------------------------------------------------------------------------------- + sleep 120 + desc 'check restore by date' + set +o xtrace ----------------------------------------------------------------------------------- check restore by date ----------------------------------------------------------------------------------- + '[' -z backupSource ']' ++ get_backup_dest backup-minio-2 ++ local backup_name=backup-minio-2 ++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|azure://||' ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.DizsZ39zg1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.jUBd3fxaBz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DizsZ39zg1 ++ cat /tmp/tmp.jUBd3fxaBz ++ rm /tmp/tmp.DizsZ39zg1 /tmp/tmp.jUBd3fxaBz ++ return 0 + backup_dest=operator-testing/pitr-prefix-1/2026-03-26T09:43:19Z + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-2/' + /usr/sbin/sed -e /backupName/d + /usr/sbin/sed -e 's/clusterName:/clusterName: some-name/' + /usr/sbin/sed -e 's/pitrType:/type: date/' + /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/pitr-prefix-1/2026-03-26T09:43:19Z|' + '[' -z '2026-03-26 09:43:48' ']' + /usr/sbin/sed -e 's/date:/date: 2026-03-26 09:43:48/' + '[' -n '' ']' + yq + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.RD9DQ4O78D ++ mktemp + local LAST_ERR=/tmp/tmp.WJCNxrYC0Y + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RD9DQ4O78D perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2 created + cat /tmp/tmp.WJCNxrYC0Y + rm /tmp/tmp.RD9DQ4O78D /tmp/tmp.WJCNxrYC0Y + return 0 + wait_restore backup-minio-2 some-name + local backup_name=backup-minio-2 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK Waiting psmdb-restore/restore-backup-minio-2 to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TDljNqVu9d +++ mktemp ++ local LAST_ERR=/tmp/tmp.o0KFix9RDy ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TDljNqVu9d ++ cat /tmp/tmp.o0KFix9RDy ++ rm /tmp/tmp.TDljNqVu9d /tmp/tmp.o0KFix9RDy ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + echo + set -o xtrace + '[' -n '' ']' + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 '' + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:47:19+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.56khJek7vb +++ mktemp ++ local LAST_ERR=/tmp/tmp.vZwuksJTTM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.56khJek7vb ++ cat /tmp/tmp.vZwuksJTTM ++ rm /tmp/tmp.56khJek7vb /tmp/tmp.vZwuksJTTM ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.FmzYaoYqfJ ++ mktemp + local LAST_ERR=/tmp/tmp.539mAV6jx3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FmzYaoYqfJ + cat /tmp/tmp.539mAV6jx3 + rm /tmp/tmp.FmzYaoYqfJ /tmp/tmp.539mAV6jx3 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find.json /tmp/tmp.mG2dpeMM56/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 '' + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:47:22+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 mongodb '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8VtW1UuLtV +++ mktemp ++ local LAST_ERR=/tmp/tmp.VVOFPDXt8X ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8VtW1UuLtV ++ cat /tmp/tmp.VVOFPDXt8X ++ rm /tmp/tmp.8VtW1UuLtV /tmp/tmp.VVOFPDXt8X ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.KoGh8I92pn ++ mktemp + local LAST_ERR=/tmp/tmp.mNiTmW27YJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KoGh8I92pn + cat /tmp/tmp.mNiTmW27YJ + rm /tmp/tmp.KoGh8I92pn /tmp/tmp.mNiTmW27YJ + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find.json /tmp/tmp.mG2dpeMM56/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 '' + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:47:24+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sWKd6relQN +++ mktemp ++ local LAST_ERR=/tmp/tmp.titsYZ8caK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.sWKd6relQN ++ cat /tmp/tmp.titsYZ8caK ++ rm /tmp/tmp.sWKd6relQN /tmp/tmp.titsYZ8caK ++ return 0 + local client_container=psmdb-client-bb8b97679-zsz7m + kubectl_bin exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.mcwmJelMEL ++ mktemp + local LAST_ERR=/tmp/tmp.3lGnd8gNFL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-zsz7m -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mcwmJelMEL + cat /tmp/tmp.3lGnd8gNFL + rm /tmp/tmp.mcwmJelMEL /tmp/tmp.3lGnd8gNFL + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/compare/find.json /tmp/tmp.mG2dpeMM56/find + run_backup backup-minio 3 + local name=backup-minio + local idx=3 + desc 'run backup backup-minio-3' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-3 ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2272/e2e-tests/pitr/conf/backup-minio.yml + /usr/sbin/sed -e 's/name:/name: backup-minio-3/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.vvS7fUZpA9 ++ mktemp + local LAST_ERR=/tmp/tmp.EVEgchfC8Z + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vvS7fUZpA9 perconaservermongodbbackup.psmdb.percona.com/backup-minio-3 created + cat /tmp/tmp.EVEgchfC8Z + rm /tmp/tmp.vvS7fUZpA9 /tmp/tmp.EVEgchfC8Z + return 0 + wait_backup backup-minio-3 + local backup_name=backup-minio-3 + local target_state=ready + set +o xtrace waiting for backup-minio-3 to reach ready state.2026-03-26T09:39:00.903Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "282b7806-6795-4714-bc33-16b1e54f8dd5", "currentJob": {"Name":"backup-minio-1","Type":0}} 2026-03-26T09:39:00.943Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "282b7806-6795-4714-bc33-16b1e54f8dd5"} 2026-03-26T09:39:00.943Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "282b7806-6795-4714-bc33-16b1e54f8dd5", "pbmName": "2026-03-26T09:39:00Z", "backup": "backup-minio-1"} 2026-03-26T09:39:00.945Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "282b7806-6795-4714-bc33-16b1e54f8dd5"} 2026-03-26T09:39:00.945Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "4a28f806-5619-4a72-89b2-b438595de024"} 2026-03-26T09:39:00.957Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "4a28f806-5619-4a72-89b2-b438595de024", "state": "requested"} 2026-03-26T09:39:00.999Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "4a28f806-5619-4a72-89b2-b438595de024", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:39:00.999Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "4a28f806-5619-4a72-89b2-b438595de024", "currentJob": {"Name":"backup-minio-1","Type":0}} 2026-03-26T09:39:01.044Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "4a28f806-5619-4a72-89b2-b438595de024"} 2026-03-26T09:39:01.045Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "4a28f806-5619-4a72-89b2-b438595de024", "pbmName": "2026-03-26T09:39:00Z", "backup": "backup-minio-1"} 2026-03-26T09:39:01.046Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "4a28f806-5619-4a72-89b2-b438595de024"} 2026-03-26T09:39:06.047Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "8a0d736a-a48a-4814-b04c-6746f0fe49e2"} 2026-03-26T09:39:06.062Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "8a0d736a-a48a-4814-b04c-6746f0fe49e2", "state": "requested"} 2026-03-26T09:39:06.102Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "8a0d736a-a48a-4814-b04c-6746f0fe49e2", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:39:06.102Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "8a0d736a-a48a-4814-b04c-6746f0fe49e2", "currentJob": {"Name":"backup-minio-1","Type":0}} 2026-03-26T09:39:06.145Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "8a0d736a-a48a-4814-b04c-6746f0fe49e2"} 2026-03-26T09:39:06.146Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "8a0d736a-a48a-4814-b04c-6746f0fe49e2", "backup": "backup-minio-1", "pbmName": "2026-03-26T09:39:00Z", "meta": {"type":"logical","opid":"69c4feb40d40528af3b65626","name":"2026-03-26T09:39:00Z","replsets":[{"name":"rs0","backup_name":"2026-03-26T09:39:00Z/rs0/metadata.json","oplog_name":"2026-03-26T09:39:00Z/rs0/oplog","start_ts":1774517942,"status":"dumpDone","size":0,"size_uncompressed":0,"last_transition_ts":1774517945,"first_write_ts":{"T":1774517942,"I":4},"last_write_ts":{"T":1774517946,"I":3},"node":"some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017","conditions":[{"timestamp":1774517942,"status":"running"},{"timestamp":1774517945,"status":"dumpDone"}],"pbm_version":"2.13.0","mongo_version":"8.0.19-7"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","prefix":"pitr-prefix-1","credentials":{"access-key-id":"***","secret-access-key":"***","vault":{}},"maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":64500,"size_uncompressed":0,"mongodb_version":"8.0.19-7","fcv":"8.0","start_ts":1774517941,"last_transition_ts":1774517946,"first_write_ts":{"T":1774517942,"I":4},"last_write_ts":{"T":1774517946,"I":3},"hb":{"T":1774517942,"I":4},"status":"dumpDone","conditions":[{"timestamp":1774517941,"status":"starting"},{"timestamp":1774517944,"status":"running"},{"timestamp":1774517946,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017"}],"pbm_version":"2.13.0","balancer":"off"}} 2026-03-26T09:39:06.148Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "8a0d736a-a48a-4814-b04c-6746f0fe49e2", "previous": "requested", "current": "running"} 2026-03-26T09:39:06.169Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "8a0d736a-a48a-4814-b04c-6746f0fe49e2"} 2026-03-26T09:39:06.179Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "6c44e5af-a4f8-4148-b063-8014e34d640d"} 2026-03-26T09:39:06.189Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "6c44e5af-a4f8-4148-b063-8014e34d640d", "state": "running"} 2026-03-26T09:39:06.234Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "6c44e5af-a4f8-4148-b063-8014e34d640d", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:39:06.234Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "6c44e5af-a4f8-4148-b063-8014e34d640d", "currentJob": {"Name":"backup-minio-1","Type":0}} 2026-03-26T09:39:06.271Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "6c44e5af-a4f8-4148-b063-8014e34d640d"} 2026-03-26T09:39:06.272Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "6c44e5af-a4f8-4148-b063-8014e34d640d", "backup": "backup-minio-1", "pbmName": "2026-03-26T09:39:00Z", "meta": {"type":"logical","opid":"69c4feb40d40528af3b65626","name":"2026-03-26T09:39:00Z","replsets":[{"name":"rs0","backup_name":"2026-03-26T09:39:00Z/rs0/metadata.json","oplog_name":"2026-03-26T09:39:00Z/rs0/oplog","start_ts":1774517942,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1774517946,"first_write_ts":{"T":1774517942,"I":4},"last_write_ts":{"T":1774517946,"I":3},"node":"some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017","conditions":[{"timestamp":1774517942,"status":"running"},{"timestamp":1774517945,"status":"dumpDone"},{"timestamp":1774517946,"status":"done"}],"pbm_version":"2.13.0","mongo_version":"8.0.19-7"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","prefix":"pitr-prefix-1","credentials":{"access-key-id":"***","secret-access-key":"***","vault":{}},"maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":64500,"size_uncompressed":0,"mongodb_version":"8.0.19-7","fcv":"8.0","start_ts":1774517941,"last_transition_ts":1774517946,"first_write_ts":{"T":1774517942,"I":4},"last_write_ts":{"T":1774517946,"I":3},"hb":{"T":1774517942,"I":4},"status":"dumpDone","conditions":[{"timestamp":1774517941,"status":"starting"},{"timestamp":1774517944,"status":"running"},{"timestamp":1774517946,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017"}],"pbm_version":"2.13.0","balancer":"off"}} 2026-03-26T09:39:06.274Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "6c44e5af-a4f8-4148-b063-8014e34d640d"} 2026-03-26T09:39:11.275Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "c03e55b2-b73c-44c6-8bb0-ee1217f16df1"} 2026-03-26T09:39:11.291Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "c03e55b2-b73c-44c6-8bb0-ee1217f16df1", "state": "running"} 2026-03-26T09:39:11.328Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "c03e55b2-b73c-44c6-8bb0-ee1217f16df1", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:39:11.328Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "c03e55b2-b73c-44c6-8bb0-ee1217f16df1", "currentJob": {"Name":"backup-minio-1","Type":0}} 2026-03-26T09:39:11.363Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "c03e55b2-b73c-44c6-8bb0-ee1217f16df1"} 2026-03-26T09:39:11.364Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "c03e55b2-b73c-44c6-8bb0-ee1217f16df1", "backup": "backup-minio-1", "pbmName": "2026-03-26T09:39:00Z", "meta": {"type":"logical","opid":"69c4feb40d40528af3b65626","name":"2026-03-26T09:39:00Z","replsets":[{"name":"rs0","backup_name":"2026-03-26T09:39:00Z/rs0/metadata.json","oplog_name":"2026-03-26T09:39:00Z/rs0/oplog","start_ts":1774517942,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1774517946,"first_write_ts":{"T":1774517942,"I":4},"last_write_ts":{"T":1774517946,"I":3},"node":"some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017","conditions":[{"timestamp":1774517942,"status":"running"},{"timestamp":1774517945,"status":"dumpDone"},{"timestamp":1774517946,"status":"done"}],"pbm_version":"2.13.0","mongo_version":"8.0.19-7"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","prefix":"pitr-prefix-1","credentials":{"access-key-id":"***","secret-access-key":"***","vault":{}},"maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":64500,"size_uncompressed":0,"mongodb_version":"8.0.19-7","fcv":"8.0","start_ts":1774517941,"last_transition_ts":1774517947,"first_write_ts":{"T":1774517942,"I":4},"last_write_ts":{"T":1774517946,"I":3},"hb":{"T":1774517947,"I":1},"status":"done","conditions":[{"timestamp":1774517941,"status":"starting"},{"timestamp":1774517944,"status":"running"},{"timestamp":1774517946,"status":"dumpDone"},{"timestamp":1774517947,"status":"done"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017"}],"pbm_version":"2.13.0","balancer":"off"}} 2026-03-26T09:39:11.366Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "c03e55b2-b73c-44c6-8bb0-ee1217f16df1", "previous": "running", "current": "ready"} 2026-03-26T09:39:11.387Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "c03e55b2-b73c-44c6-8bb0-ee1217f16df1", "lease": "psmdb-some-name-backup-lock"} 2026-03-26T09:39:11.418Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "c03e55b2-b73c-44c6-8bb0-ee1217f16df1"} 2026-03-26T09:39:11.418Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "52f20e66-b54c-4afa-8fc0-7f578a991c84"} 2026-03-26T09:39:11.432Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "52f20e66-b54c-4afa-8fc0-7f578a991c84", "state": "ready"} 2026-03-26T09:39:11.432Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "52f20e66-b54c-4afa-8fc0-7f578a991c84"} 2026-03-26T09:39:18.200Z INFO PBM updating latest restorable time {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "some-name", "reconcileID": "d1fffdea-f3e3-4265-8af1-d6452fad667b", "backup": "backup-minio-1", "latestRestorableTime": "2026-03-26 09:38:29 +0000 UTC"} 2026-03-26T09:39:18.245Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "cc0bd9b9-a2db-4613-bbac-ff6f4830cf8e"} 2026-03-26T09:39:18.262Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "cc0bd9b9-a2db-4613-bbac-ff6f4830cf8e", "state": "ready"} 2026-03-26T09:39:18.262Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "cc0bd9b9-a2db-4613-bbac-ff6f4830cf8e"} 2026-03-26T09:41:38.680Z INFO PBM updating latest restorable time {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "some-name", "reconcileID": "5c698d07-92a5-49b5-802e-b13867f6feb5", "backup": "backup-minio-1", "latestRestorableTime": "2026-03-26 09:41:35 +0000 UTC"} 2026-03-26T09:41:38.733Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "c3d61e2a-f067-4846-9aac-10dd2da3296b"} 2026-03-26T09:41:38.750Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "c3d61e2a-f067-4846-9aac-10dd2da3296b", "state": "ready"} 2026-03-26T09:41:38.750Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "c3d61e2a-f067-4846-9aac-10dd2da3296b"} 2026-03-26T09:42:32.158Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "1d060a81-23c7-4b2d-bba0-61e31470d564", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:42:32.302Z INFO Waiting for PITR to be disabled. {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "1d060a81-23c7-4b2d-bba0-61e31470d564"} 2026-03-26T09:42:32.304Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "1d060a81-23c7-4b2d-bba0-61e31470d564", "previous": "", "current": "waiting"} 2026-03-26T09:42:32.848Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "d75b2bca-9d40-4529-b83f-4bff9fe8b22b", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:42:32.889Z INFO Waiting for PITR to be disabled. {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "d75b2bca-9d40-4529-b83f-4bff9fe8b22b"} 2026-03-26T09:42:37.892Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "d42a209f-4d20-411a-b711-ae0617d97201", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:42:37.936Z INFO Starting restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "d42a209f-4d20-411a-b711-ae0617d97201", "backup": "2026-03-26T09:39:00Z"} 2026-03-26T09:42:37.936Z INFO Starting logical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "d42a209f-4d20-411a-b711-ae0617d97201", "backup": "2026-03-26T09:39:00Z"} 2026-03-26T09:42:37.938Z INFO Sending restore command {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "d42a209f-4d20-411a-b711-ae0617d97201", "restoreCmd": "name: 2026-03-26T09:42:37.936787161Z, snapshot: 2026-03-26T09:39:00Z point-in-time: <1774518155,0>"} 2026-03-26T09:42:37.943Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "d42a209f-4d20-411a-b711-ae0617d97201", "previous": "waiting", "current": "requested"} 2026-03-26T09:42:38.493Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "03693f40-f7fa-4356-ac4e-b4f07a62a1bb", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:42:38.533Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "03693f40-f7fa-4356-ac4e-b4f07a62a1bb", "previous": "requested", "current": "running"} 2026-03-26T09:42:39.070Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "060cec56-6657-4fa3-8ac2-1b9b0db1cb05", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:42:44.107Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "93b25d07-0ad0-4cbe-9465-7d52030e3c6e", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:42:49.154Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "77d01def-fa20-4a16-b966-e943fd09602b", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:42:49.205Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-1", "reconcileID": "77d01def-fa20-4a16-b966-e943fd09602b", "previous": "running", "current": "ready"} 2026-03-26T09:42:49.933Z INFO PBM updating latest restorable time {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "some-name", "reconcileID": "2c09908d-80b1-428c-b3f1-28f4c7d9527b", "backup": "backup-minio-1", "latestRestorableTime": "2026-03-26 09:42:35 +0000 UTC"} 2026-03-26T09:42:49.990Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "447ac6aa-d5f6-4395-80e7-14c7a59ef31c"} 2026-03-26T09:42:50.003Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "447ac6aa-d5f6-4395-80e7-14c7a59ef31c", "state": "ready"} 2026-03-26T09:42:50.003Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-1", "reconcileID": "447ac6aa-d5f6-4395-80e7-14c7a59ef31c"} 2026-03-26T09:43:19.654Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "bcff249f-8c36-4984-9b1e-ad49bbc11aa4"} 2026-03-26T09:43:19.668Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "bcff249f-8c36-4984-9b1e-ad49bbc11aa4", "state": ""} 2026-03-26T09:43:19.709Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "bcff249f-8c36-4984-9b1e-ad49bbc11aa4", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:43:19.709Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "bcff249f-8c36-4984-9b1e-ad49bbc11aa4", "currentJob": {"Name":"backup-minio-2","Type":0}} 2026-03-26T09:43:19.746Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "bcff249f-8c36-4984-9b1e-ad49bbc11aa4"} 2026-03-26T09:43:19.781Z DEBUG Added finalizer {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "bcff249f-8c36-4984-9b1e-ad49bbc11aa4", "finalizer": "internal.percona.com/release-lock"} 2026-03-26T09:43:19.781Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "bcff249f-8c36-4984-9b1e-ad49bbc11aa4", "backup": "backup-minio-2", "storage": "minio"} 2026-03-26T09:43:19.781Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "bcff249f-8c36-4984-9b1e-ad49bbc11aa4", "backup": "backup-minio-2", "storage": "minio", "backupCmd": "backup [name: 2026-03-26T09:43:19Z, compression: gzip (level: default)] ", "profile": ""} 2026-03-26T09:43:19.788Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "bcff249f-8c36-4984-9b1e-ad49bbc11aa4", "previous": "", "current": "requested"} 2026-03-26T09:43:19.835Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "bcff249f-8c36-4984-9b1e-ad49bbc11aa4"} 2026-03-26T09:43:19.835Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "28a36207-535a-4096-a761-fb1fb01a4458"} 2026-03-26T09:43:19.848Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "28a36207-535a-4096-a761-fb1fb01a4458", "state": "requested"} 2026-03-26T09:43:19.886Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "28a36207-535a-4096-a761-fb1fb01a4458", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:43:19.886Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "28a36207-535a-4096-a761-fb1fb01a4458", "currentJob": {"Name":"backup-minio-2","Type":0}} 2026-03-26T09:43:19.921Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "28a36207-535a-4096-a761-fb1fb01a4458"} 2026-03-26T09:43:19.922Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "28a36207-535a-4096-a761-fb1fb01a4458", "pbmName": "2026-03-26T09:43:19Z", "backup": "backup-minio-2"} 2026-03-26T09:43:19.923Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "28a36207-535a-4096-a761-fb1fb01a4458"} 2026-03-26T09:43:19.923Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "c87ff673-8e61-46f4-951e-93bdcb2209dd"} 2026-03-26T09:43:19.936Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "c87ff673-8e61-46f4-951e-93bdcb2209dd", "state": "requested"} 2026-03-26T09:43:19.970Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "c87ff673-8e61-46f4-951e-93bdcb2209dd", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:43:19.970Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "c87ff673-8e61-46f4-951e-93bdcb2209dd", "currentJob": {"Name":"backup-minio-2","Type":0}} 2026-03-26T09:43:20.007Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "c87ff673-8e61-46f4-951e-93bdcb2209dd"} 2026-03-26T09:43:20.008Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "c87ff673-8e61-46f4-951e-93bdcb2209dd", "pbmName": "2026-03-26T09:43:19Z", "backup": "backup-minio-2"} 2026-03-26T09:43:20.009Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "c87ff673-8e61-46f4-951e-93bdcb2209dd"} 2026-03-26T09:43:25.010Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "65504599-d687-420b-8032-dd51a08dde5e"} 2026-03-26T09:43:25.043Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "65504599-d687-420b-8032-dd51a08dde5e", "state": "requested"} 2026-03-26T09:43:25.082Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "65504599-d687-420b-8032-dd51a08dde5e", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:43:25.082Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "65504599-d687-420b-8032-dd51a08dde5e", "currentJob": {"Name":"backup-minio-2","Type":0}} 2026-03-26T09:43:25.121Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "65504599-d687-420b-8032-dd51a08dde5e"} 2026-03-26T09:43:25.121Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "65504599-d687-420b-8032-dd51a08dde5e", "backup": "backup-minio-2", "pbmName": "2026-03-26T09:43:19Z", "meta": {"type":"logical","opid":"69c4ffb70d40528af3b656ec","name":"2026-03-26T09:43:19Z","replsets":[{"name":"rs0","backup_name":"2026-03-26T09:43:19Z/rs0/metadata.json","oplog_name":"2026-03-26T09:43:19Z/rs0/oplog","start_ts":1774518200,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1774518204,"first_write_ts":{"T":1774518200,"I":20},"last_write_ts":{"T":1774518204,"I":2},"node":"some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017","conditions":[{"timestamp":1774518200,"status":"running"},{"timestamp":1774518203,"status":"dumpDone"},{"timestamp":1774518204,"status":"done"}],"pbm_version":"2.13.0","mongo_version":"8.0.19-7"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","prefix":"pitr-prefix-1","credentials":{"access-key-id":"***","secret-access-key":"***","vault":{}},"maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":64543,"size_uncompressed":0,"mongodb_version":"8.0.19-7","fcv":"8.0","start_ts":1774518200,"last_transition_ts":1774518204,"first_write_ts":{"T":1774518200,"I":20},"last_write_ts":{"T":1774518204,"I":2},"hb":{"T":1774518200,"I":20},"status":"dumpDone","conditions":[{"timestamp":1774518200,"status":"starting"},{"timestamp":1774518202,"status":"running"},{"timestamp":1774518204,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017"}],"pbm_version":"2.13.0","balancer":"off"}} 2026-03-26T09:43:25.123Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "65504599-d687-420b-8032-dd51a08dde5e", "previous": "requested", "current": "running"} 2026-03-26T09:43:25.146Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "65504599-d687-420b-8032-dd51a08dde5e"} 2026-03-26T09:43:25.152Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "1cf469b6-ba3b-41ad-b776-ba179fdc1b08"} 2026-03-26T09:43:25.169Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "1cf469b6-ba3b-41ad-b776-ba179fdc1b08", "state": "running"} 2026-03-26T09:43:25.209Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "1cf469b6-ba3b-41ad-b776-ba179fdc1b08", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:43:25.209Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "1cf469b6-ba3b-41ad-b776-ba179fdc1b08", "currentJob": {"Name":"backup-minio-2","Type":0}} 2026-03-26T09:43:25.259Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "1cf469b6-ba3b-41ad-b776-ba179fdc1b08"} 2026-03-26T09:43:25.260Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "1cf469b6-ba3b-41ad-b776-ba179fdc1b08", "backup": "backup-minio-2", "pbmName": "2026-03-26T09:43:19Z", "meta": {"type":"logical","opid":"69c4ffb70d40528af3b656ec","name":"2026-03-26T09:43:19Z","replsets":[{"name":"rs0","backup_name":"2026-03-26T09:43:19Z/rs0/metadata.json","oplog_name":"2026-03-26T09:43:19Z/rs0/oplog","start_ts":1774518200,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1774518204,"first_write_ts":{"T":1774518200,"I":20},"last_write_ts":{"T":1774518204,"I":2},"node":"some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017","conditions":[{"timestamp":1774518200,"status":"running"},{"timestamp":1774518203,"status":"dumpDone"},{"timestamp":1774518204,"status":"done"}],"pbm_version":"2.13.0","mongo_version":"8.0.19-7"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","prefix":"pitr-prefix-1","credentials":{"access-key-id":"***","secret-access-key":"***","vault":{}},"maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":64543,"size_uncompressed":0,"mongodb_version":"8.0.19-7","fcv":"8.0","start_ts":1774518200,"last_transition_ts":1774518204,"first_write_ts":{"T":1774518200,"I":20},"last_write_ts":{"T":1774518204,"I":2},"hb":{"T":1774518200,"I":20},"status":"dumpDone","conditions":[{"timestamp":1774518200,"status":"starting"},{"timestamp":1774518202,"status":"running"},{"timestamp":1774518204,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017"}],"pbm_version":"2.13.0","balancer":"off"}} 2026-03-26T09:43:25.262Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "1cf469b6-ba3b-41ad-b776-ba179fdc1b08"} 2026-03-26T09:43:30.263Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "527aedca-4eef-402d-97c1-4d378dee3c5f"} 2026-03-26T09:43:30.274Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "527aedca-4eef-402d-97c1-4d378dee3c5f", "state": "running"} 2026-03-26T09:43:30.314Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "527aedca-4eef-402d-97c1-4d378dee3c5f", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:43:30.314Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "527aedca-4eef-402d-97c1-4d378dee3c5f", "currentJob": {"Name":"backup-minio-2","Type":0}} 2026-03-26T09:43:30.350Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "527aedca-4eef-402d-97c1-4d378dee3c5f"} 2026-03-26T09:43:30.351Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "527aedca-4eef-402d-97c1-4d378dee3c5f", "backup": "backup-minio-2", "pbmName": "2026-03-26T09:43:19Z", "meta": {"type":"logical","opid":"69c4ffb70d40528af3b656ec","name":"2026-03-26T09:43:19Z","replsets":[{"name":"rs0","backup_name":"2026-03-26T09:43:19Z/rs0/metadata.json","oplog_name":"2026-03-26T09:43:19Z/rs0/oplog","start_ts":1774518200,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1774518204,"first_write_ts":{"T":1774518200,"I":20},"last_write_ts":{"T":1774518204,"I":2},"node":"some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017","conditions":[{"timestamp":1774518200,"status":"running"},{"timestamp":1774518203,"status":"dumpDone"},{"timestamp":1774518204,"status":"done"}],"pbm_version":"2.13.0","mongo_version":"8.0.19-7"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","prefix":"pitr-prefix-1","credentials":{"access-key-id":"***","secret-access-key":"***","vault":{}},"maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":64543,"size_uncompressed":0,"mongodb_version":"8.0.19-7","fcv":"8.0","start_ts":1774518200,"last_transition_ts":1774518205,"first_write_ts":{"T":1774518200,"I":20},"last_write_ts":{"T":1774518204,"I":2},"hb":{"T":1774518200,"I":20},"status":"done","conditions":[{"timestamp":1774518200,"status":"starting"},{"timestamp":1774518202,"status":"running"},{"timestamp":1774518204,"status":"dumpDone"},{"timestamp":1774518205,"status":"done"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.pitr-24754.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.pitr-24754.svc.cluster.local:27017"}],"pbm_version":"2.13.0","balancer":"off"}} 2026-03-26T09:43:30.353Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "527aedca-4eef-402d-97c1-4d378dee3c5f", "previous": "running", "current": "ready"} 2026-03-26T09:43:30.375Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "527aedca-4eef-402d-97c1-4d378dee3c5f", "lease": "psmdb-some-name-backup-lock"} 2026-03-26T09:43:30.404Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "527aedca-4eef-402d-97c1-4d378dee3c5f"} 2026-03-26T09:43:30.405Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "d864d778-3d26-42d7-92cf-a5351f2893ab"} 2026-03-26T09:43:30.417Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "d864d778-3d26-42d7-92cf-a5351f2893ab", "state": "ready"} 2026-03-26T09:43:30.417Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "d864d778-3d26-42d7-92cf-a5351f2893ab"} 2026-03-26T09:43:33.711Z INFO PBM updating latest restorable time {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "some-name", "reconcileID": "2f4ae44e-b94a-489c-ae92-6a4ab52dd165", "backup": "backup-minio-2", "latestRestorableTime": "2026-03-26 09:42:35 +0000 UTC"} 2026-03-26T09:43:33.753Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "5013ad67-e131-4f78-9921-59f38dadf6d0"} 2026-03-26T09:43:33.768Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "5013ad67-e131-4f78-9921-59f38dadf6d0", "state": "ready"} 2026-03-26T09:43:33.769Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "5013ad67-e131-4f78-9921-59f38dadf6d0"} 2026-03-26T09:46:02.359Z INFO PBM updating latest restorable time {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "some-name", "reconcileID": "1e11496b-53a8-4860-a0e1-34e0aa8789cb", "backup": "backup-minio-2", "latestRestorableTime": "2026-03-26 09:45:56 +0000 UTC"} 2026-03-26T09:46:02.411Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "7365ceaa-2752-4c2b-a9d6-d3226831d56b"} 2026-03-26T09:46:02.424Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "7365ceaa-2752-4c2b-a9d6-d3226831d56b", "state": "ready"} 2026-03-26T09:46:02.424Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "7365ceaa-2752-4c2b-a9d6-d3226831d56b"} 2026-03-26T09:46:54.218Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "bdb20150-2d5b-44b9-a8d7-d6ba245d81f6", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:46:54.353Z INFO Waiting for PITR to be disabled. {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "bdb20150-2d5b-44b9-a8d7-d6ba245d81f6"} 2026-03-26T09:46:54.354Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "bdb20150-2d5b-44b9-a8d7-d6ba245d81f6", "previous": "", "current": "waiting"} 2026-03-26T09:46:55.042Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "9af7e36c-391a-4e02-b5e2-acfb05dfd5ce", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:46:55.084Z INFO Waiting for PITR to be disabled. {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "9af7e36c-391a-4e02-b5e2-acfb05dfd5ce"} 2026-03-26T09:47:00.086Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "8b87e820-c2ef-4dbd-b7a8-adb869157550", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:47:00.134Z INFO Starting restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "8b87e820-c2ef-4dbd-b7a8-adb869157550", "backup": "2026-03-26T09:43:19Z"} 2026-03-26T09:47:00.134Z INFO Starting logical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "8b87e820-c2ef-4dbd-b7a8-adb869157550", "backup": "2026-03-26T09:43:19Z"} 2026-03-26T09:47:00.135Z INFO Sending restore command {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "8b87e820-c2ef-4dbd-b7a8-adb869157550", "restoreCmd": "name: 2026-03-26T09:47:00.134597554Z, snapshot: 2026-03-26T09:43:19Z point-in-time: <1774518228,0>"} 2026-03-26T09:47:00.139Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "8b87e820-c2ef-4dbd-b7a8-adb869157550", "previous": "waiting", "current": "requested"} 2026-03-26T09:47:00.710Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "f9c4d904-d5ec-4db0-b6fe-3df14a7b631b", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:47:00.746Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "f9c4d904-d5ec-4db0-b6fe-3df14a7b631b", "previous": "requested", "current": "running"} 2026-03-26T09:47:01.329Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "67e2eaf7-6f8d-423d-9912-6bd22e5f1805", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:47:06.366Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "70a36dfc-075f-4e69-8c43-cf2b4a0f3400", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:47:11.413Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "f6bdc32a-0cc3-47b5-817c-661ba9c043ab", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:47:11.455Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "restore-backup-minio-2", "reconcileID": "f6bdc32a-0cc3-47b5-817c-661ba9c043ab", "previous": "running", "current": "ready"} 2026-03-26T09:47:14.410Z INFO PBM updating latest restorable time {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "some-name", "reconcileID": "c722a329-9a45-4303-b353-2ca2e9025ec3", "backup": "backup-minio-2", "latestRestorableTime": "2026-03-26 09:46:56 +0000 UTC"} 2026-03-26T09:47:14.459Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "7e754593-cd14-49c0-899d-dde251088ddb"} 2026-03-26T09:47:14.477Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "7e754593-cd14-49c0-899d-dde251088ddb", "state": "ready"} 2026-03-26T09:47:14.477Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-2", "reconcileID": "7e754593-cd14-49c0-899d-dde251088ddb"} 2026-03-26T09:47:29.596Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "8f51942b-d398-47c9-ad7c-791401df0917"} 2026-03-26T09:47:29.640Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "8f51942b-d398-47c9-ad7c-791401df0917", "state": ""} 2026-03-26T09:47:29.678Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "8f51942b-d398-47c9-ad7c-791401df0917", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:47:29.678Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "8f51942b-d398-47c9-ad7c-791401df0917", "currentJob": {"Name":"backup-minio-3","Type":0}} 2026-03-26T09:47:29.721Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "8f51942b-d398-47c9-ad7c-791401df0917"} 2026-03-26T09:47:29.765Z DEBUG Added finalizer {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "8f51942b-d398-47c9-ad7c-791401df0917", "finalizer": "internal.percona.com/release-lock"} 2026-03-26T09:47:29.765Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "8f51942b-d398-47c9-ad7c-791401df0917", "backup": "backup-minio-3", "storage": "minio"} 2026-03-26T09:47:29.765Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "8f51942b-d398-47c9-ad7c-791401df0917", "backup": "backup-minio-3", "storage": "minio", "backupCmd": "backup [name: 2026-03-26T09:47:29Z, compression: gzip (level: default)] ", "profile": ""} 2026-03-26T09:47:29.772Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "8f51942b-d398-47c9-ad7c-791401df0917", "previous": "", "current": "requested"} 2026-03-26T09:47:29.905Z ERROR failed to update backup status {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "8f51942b-d398-47c9-ad7c-791401df0917", "backup": "backup-minio-3", "error": "write status: Operation cannot be fulfilled on perconaservermongodbbackups.psmdb.percona.com \"backup-minio-3\": the object has been modified; please apply your changes to the latest version and try again", "errorVerbose": "Operation cannot be fulfilled on perconaservermongodbbackups.psmdb.percona.com \"backup-minio-3\": the object has been modified; please apply your changes to the latest version and try again\nwrite status\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).updateStatus\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:763\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:153\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:155 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-26T09:47:29.906Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "8f51942b-d398-47c9-ad7c-791401df0917"} 2026-03-26T09:47:30.173Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "a6673810-4bae-449f-a49d-aef0b31dfa82"} 2026-03-26T09:47:30.185Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "a6673810-4bae-449f-a49d-aef0b31dfa82", "state": ""} 2026-03-26T09:47:30.229Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "a6673810-4bae-449f-a49d-aef0b31dfa82", "cluster": "some-name", "namespace": "pitr-24754"} 2026-03-26T09:47:30.229Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "a6673810-4bae-449f-a49d-aef0b31dfa82", "currentJob": {"Name":"backup-minio-3","Type":0}} 2026-03-26T09:47:30.269Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "a6673810-4bae-449f-a49d-aef0b31dfa82"} 2026-03-26T09:47:30.329Z ERROR failed to make backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "a6673810-4bae-449f-a49d-aef0b31dfa82", "backup": "backup-minio-3", "error": "acquire backup lock: create lease: leases.coordination.k8s.io \"psmdb-some-name-backup-lock\" already exists", "errorVerbose": "leases.coordination.k8s.io \"psmdb-some-name-backup-lock\" already exists\ncreate lease\ngithub.com/percona/percona-server-mongodb-operator/pkg/k8s.AcquireLease\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/k8s/lease.go:39\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:276\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nacquire backup lock\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:278\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:148 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-26T09:47:30.329Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "a6673810-4bae-449f-a49d-aef0b31dfa82", "previous": "", "current": "error"} 2026-03-26T09:47:30.350Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "a6673810-4bae-449f-a49d-aef0b31dfa82", "lease": "psmdb-some-name-backup-lock"} 2026-03-26T09:47:30.379Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "a6673810-4bae-449f-a49d-aef0b31dfa82"} 2026-03-26T09:47:30.379Z INFO Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "a6673810-4bae-449f-a49d-aef0b31dfa82"} 2026-03-26T09:47:30.379Z ERROR Reconciler error {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "a6673810-4bae-449f-a49d-aef0b31dfa82", "error": "reconcile backup: acquire backup lock: create lease: leases.coordination.k8s.io \"psmdb-some-name-backup-lock\" already exists", "errorVerbose": "leases.coordination.k8s.io \"psmdb-some-name-backup-lock\" already exists\ncreate lease\ngithub.com/percona/percona-server-mongodb-operator/pkg/k8s.AcquireLease\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/k8s/lease.go:39\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:276\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nacquire backup lock\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:278\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:239\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-03-26T09:47:30.379Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "217fcf5b-8c46-41bb-8441-280c25283823"} 2026-03-26T09:47:30.420Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "217fcf5b-8c46-41bb-8441-280c25283823", "state": "error"} 2026-03-26T09:47:30.420Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-24754"}, "namespace": "pitr-24754", "name": "backup-minio-3", "reconcileID": "217fcf5b-8c46-41bb-8441-280c25283823"} NAME CLUSTER STORAGE DESTINATION TYPE SIZE STATUS COMPLETED AGE backup-minio-0 some-name minio s3://operator-testing/pitr-prefix-1/2026-03-26T09:34:52Z logical 53.59KB ready 12m 12m backup-minio-1 some-name minio s3://operator-testing/pitr-prefix-1/2026-03-26T09:39:00Z logical 62.99KB ready 8m27s 8m34s backup-minio-2 some-name minio s3://operator-testing/pitr-prefix-1/2026-03-26T09:43:19Z logical 63.03KB ready 4m9s 4m15s backup-minio-3 some-name minio error 5s Name: backup-minio-3 Namespace: pitr-24754 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBBackup Metadata: Creation Timestamp: 2026-03-26T09:47:29Z Finalizers: percona.com/delete-backup internal.percona.com/release-lock Generation: 1 Resource Version: 1774518450343391004 UID: 80817ea2-76dc-4f0c-a53d-5ee6303bd8af Spec: Cluster Name: some-name Starting Deadline Seconds: 120 Storage Name: minio Status: Error: acquire backup lock: create lease: leases.coordination.k8s.io "psmdb-some-name-backup-lock" already exists State: error Events: Backup object psmdb-backup/backup-minio-3 is in error state.