Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/logs/pitr.log WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 + main + create_infra pitr-8397 + local ns=pitr-8397 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.IiVR9gmNeW ++ mktemp + local LAST_ERR=/tmp/tmp.5gHk6KduWh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IiVR9gmNeW customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.5gHk6KduWh + rm /tmp/tmp.IiVR9gmNeW /tmp/tmp.5gHk6KduWh + return 0 ++ grep -v '\-\-\-' ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.4z20YluMFw ++ mktemp + local LAST_ERR=/tmp/tmp.rXKSrS8OW2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4z20YluMFw + cat /tmp/tmp.rXKSrS8OW2 + rm /tmp/tmp.4z20YluMFw /tmp/tmp.rXKSrS8OW2 + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.j52bQTOa8A ++ mktemp + local LAST_ERR=/tmp/tmp.e8XxEzY0Gm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.j52bQTOa8A + cat /tmp/tmp.e8XxEzY0Gm + rm /tmp/tmp.j52bQTOa8A /tmp/tmp.e8XxEzY0Gm + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.dbCtRYDFzk ++ mktemp + local LAST_ERR=/tmp/tmp.eHtUgbvg4v + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dbCtRYDFzk + cat /tmp/tmp.eHtUgbvg4v + rm /tmp/tmp.dbCtRYDFzk /tmp/tmp.eHtUgbvg4v + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.r8Bnydzpo1 ++ mktemp + local LAST_ERR=/tmp/tmp.XAquaXkJQY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.r8Bnydzpo1 clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.XAquaXkJQY + rm /tmp/tmp.r8Bnydzpo1 /tmp/tmp.XAquaXkJQY + return 0 + check_crd_for_deletion PR-1915-2d829121 + local git_tag=PR-1915-2d829121 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1915-2d829121/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/bin/sed s/---//g ++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')' ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sSkG9K0A7u +++ mktemp ++ local LAST_ERR=/tmp/tmp.GvuzGLXM5h ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.sSkG9K0A7u ++ cat /tmp/tmp.GvuzGLXM5h Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.sSkG9K0A7u ++ cat /tmp/tmp.GvuzGLXM5h Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.sSkG9K0A7u ++ cat /tmp/tmp.GvuzGLXM5h Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.sSkG9K0A7u ++ cat /tmp/tmp.GvuzGLXM5h Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.sSkG9K0A7u /tmp/tmp.GvuzGLXM5h ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found + xargs kubectl delete ns ++ mktemp + kubectl_bin get ns + local LAST_OUT=/tmp/tmp.rqlZTZK5w1 + awk '{print$1}' + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.KwcL4k7Vln + local exit_status=0 + local timeout=4 + local LAST_OUT=/tmp/tmp.G0udUxTX6U ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace psmdb-operator --ignore-not-found ++ mktemp + local LAST_ERR=/tmp/tmp.pAaLXdEEEK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.G0udUxTX6U + cat /tmp/tmp.pAaLXdEEEK + rm /tmp/tmp.G0udUxTX6U /tmp/tmp.pAaLXdEEEK + return 0 namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted namespace "pitr-1834" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rqlZTZK5w1 namespace "psmdb-operator" deleted + cat /tmp/tmp.KwcL4k7Vln + rm /tmp/tmp.rqlZTZK5w1 /tmp/tmp.KwcL4k7Vln + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.dTOBWRyQRR ++ mktemp + local LAST_ERR=/tmp/tmp.beqjMkugPE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dTOBWRyQRR + cat /tmp/tmp.beqjMkugPE + rm /tmp/tmp.dTOBWRyQRR /tmp/tmp.beqjMkugPE + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.IDEsvFiEpp ++ mktemp + local LAST_ERR=/tmp/tmp.JAVuHNCKia + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IDEsvFiEpp namespace/psmdb-operator created + cat /tmp/tmp.JAVuHNCKia + rm /tmp/tmp.IDEsvFiEpp /tmp/tmp.JAVuHNCKia + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.RaWW0nhbrF +++ mktemp ++ local LAST_ERR=/tmp/tmp.pjyd8VwKuS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RaWW0nhbrF ++ cat /tmp/tmp.pjyd8VwKuS ++ rm /tmp/tmp.RaWW0nhbrF /tmp/tmp.pjyd8VwKuS ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster9 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.IIn9AzCpbL ++ mktemp + local LAST_ERR=/tmp/tmp.SmSpBa1y17 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster9 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IIn9AzCpbL Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster9" modified. + cat /tmp/tmp.SmSpBa1y17 + rm /tmp/tmp.IIn9AzCpbL /tmp/tmp.SmSpBa1y17 + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.2oocSXZhHI ++ mktemp + local LAST_ERR=/tmp/tmp.7QMb8Ympf5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2oocSXZhHI customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.7QMb8Ympf5 + rm /tmp/tmp.2oocSXZhHI /tmp/tmp.7QMb8Ympf5 + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/cw-rbac.yaml + kubectl_bin apply -n psmdb-operator -f - + sed -e 's^namespace: .*^namespace: psmdb-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.AMpPpLSidI ++ mktemp + local LAST_ERR=/tmp/tmp.Lium4rlEnX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AMpPpLSidI clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.Lium4rlEnX + rm /tmp/tmp.AMpPpLSidI /tmp/tmp.Lium4rlEnX + return 0 + kubectl_bin apply -f - + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1915-2d829121") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/cw-operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.wdfZXPdvW2 ++ mktemp + local LAST_ERR=/tmp/tmp.S7WQVpfq06 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wdfZXPdvW2 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.S7WQVpfq06 + rm /tmp/tmp.wdfZXPdvW2 /tmp/tmp.S7WQVpfq06 + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.2MBeZXbtLH +++ mktemp ++ local LAST_ERR=/tmp/tmp.wFFMb2RNsr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2MBeZXbtLH ++ cat /tmp/tmp.wFFMb2RNsr ++ rm /tmp/tmp.2MBeZXbtLH /tmp/tmp.wFFMb2RNsr ++ return 0 + wait_pod percona-server-mongodb-operator-6c96ccdc79-nqlcp + local pod=percona-server-mongodb-operator-6c96ccdc79-nqlcp + set +o xtrace waiting for pod/percona-server-mongodb-operator-6c96ccdc79-nqlcp to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.JI2yFagSAL +++ mktemp ++ local LAST_ERR=/tmp/tmp.fbXjPbkWl3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JI2yFagSAL ++ cat /tmp/tmp.fbXjPbkWl3 ++ rm /tmp/tmp.JI2yFagSAL /tmp/tmp.fbXjPbkWl3 ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-6c96ccdc79-nqlcp ++ mktemp + local LAST_OUT=/tmp/tmp.R2igFg3kEA ++ mktemp + local LAST_ERR=/tmp/tmp.3WRscLRLhP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs percona-server-mongodb-operator-6c96ccdc79-nqlcp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.R2igFg3kEA + cat /tmp/tmp.3WRscLRLhP + rm /tmp/tmp.R2igFg3kEA /tmp/tmp.3WRscLRLhP + return 0 2025-05-21T10:30:17.140Z INFO setup Manager starting up {"gitCommit": "2d8291219bbb1a7ad015786c5ebd76d37f8fba63", "gitBranch": "PR-1915-2d829121", "buildTime": "", "goVersion": "go1.24.3", "os": "linux", "arch": "amd64"} + create_namespace pitr-8397 + local namespace=pitr-8397 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pitr-8397' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pitr-8397 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pitr-8397 --ignore-not-found ++ mktemp + awk '{print$1}' + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + local LAST_OUT=/tmp/tmp.fSeHP5FhuH ++ mktemp + xargs kubectl delete ns + kubectl_bin get ns ++ mktemp + local LAST_ERR=/tmp/tmp.4xa1IQRGFd + local exit_status=0 + local timeout=4 + local LAST_OUT=/tmp/tmp.3zUIkVBGS3 ++ seq 0 2 ++ mktemp + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pitr-8397 --ignore-not-found + local LAST_ERR=/tmp/tmp.yMlgmz158G + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3zUIkVBGS3 + cat /tmp/tmp.yMlgmz158G + rm /tmp/tmp.3zUIkVBGS3 /tmp/tmp.yMlgmz158G + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fSeHP5FhuH + cat /tmp/tmp.4xa1IQRGFd + rm /tmp/tmp.fSeHP5FhuH /tmp/tmp.4xa1IQRGFd + return 0 + kubectl_bin wait --for=delete namespace pitr-8397 ++ mktemp + local LAST_OUT=/tmp/tmp.4vTy2dxVXR ++ mktemp + local LAST_ERR=/tmp/tmp.Ti0HmsPN5A + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace pitr-8397 namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4vTy2dxVXR + cat /tmp/tmp.Ti0HmsPN5A + rm /tmp/tmp.4vTy2dxVXR /tmp/tmp.Ti0HmsPN5A + return 0 + desc 'create namespace pitr-8397' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pitr-8397 ----------------------------------------------------------------------------------- + kubectl_bin create namespace pitr-8397 ++ mktemp + local LAST_OUT=/tmp/tmp.p58T6aX9EZ ++ mktemp + local LAST_ERR=/tmp/tmp.OnCWQAo0Dh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pitr-8397 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.p58T6aX9EZ namespace/pitr-8397 created + cat /tmp/tmp.OnCWQAo0Dh + rm /tmp/tmp.p58T6aX9EZ /tmp/tmp.OnCWQAo0Dh + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Je5hFOJz7r +++ mktemp ++ local LAST_ERR=/tmp/tmp.QBpGBoxzIU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Je5hFOJz7r ++ cat /tmp/tmp.QBpGBoxzIU ++ rm /tmp/tmp.Je5hFOJz7r /tmp/tmp.QBpGBoxzIU ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster9 --namespace=pitr-8397 ++ mktemp + local LAST_OUT=/tmp/tmp.npwrM7mh8B ++ mktemp + local LAST_ERR=/tmp/tmp.bAyNB24cCi + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster9 --namespace=pitr-8397 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.npwrM7mh8B Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster9" modified. + cat /tmp/tmp.bAyNB24cCi + rm /tmp/tmp.npwrM7mh8B /tmp/tmp.bAyNB24cCi + return 0 + deploy_minio + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio NAME: minio-service LAST DEPLOYED: Wed May 21 10:30:59 2025 NAMESPACE: pitr-8397 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.pitr-8397.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace pitr-8397 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace pitr-8397 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-8397 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-8397 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.da3xmgJlG0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.plC5hhgZpd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.da3xmgJlG0 ++ cat /tmp/tmp.plC5hhgZpd ++ rm /tmp/tmp.da3xmgJlG0 /tmp/tmp.plC5hhgZpd ++ return 0 + MINIO_POD=minio-service-86dfccd949-qf5zg + wait_pod minio-service-86dfccd949-qf5zg + local pod=minio-service-86dfccd949-qf5zg + set +o xtrace waiting for pod/minio-service-86dfccd949-qf5zg to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-8397.svc.cluster.local --tcp=9000 ++ mktemp + local LAST_OUT=/tmp/tmp.uhqttU0vCx ++ mktemp + local LAST_ERR=/tmp/tmp.ZBzhgmaQNQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-8397.svc.cluster.local --tcp=9000 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uhqttU0vCx service/minio-service created + cat /tmp/tmp.ZBzhgmaQNQ + rm /tmp/tmp.uhqttU0vCx /tmp/tmp.ZBzhgmaQNQ + return 0 + create_minio_bucket operator-testing + local bucket=operator-testing + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.4TBzn8l6OR ++ mktemp + local LAST_ERR=/tmp/tmp.mOhcFiwCKu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4TBzn8l6OR pod "aws-cli" deleted + cat /tmp/tmp.mOhcFiwCKu If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.4TBzn8l6OR /tmp/tmp.mOhcFiwCKu + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/minio-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.BSeR42PrvX ++ mktemp + local LAST_ERR=/tmp/tmp.QItdWVNepo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/minio-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BSeR42PrvX secret/some-users created deployment.apps/psmdb-client created secret/minio-secret created + cat /tmp/tmp.QItdWVNepo + rm /tmp/tmp.BSeR42PrvX /tmp/tmp.QItdWVNepo + return 0 + cluster=some-name-rs0 + desc 'create first PSMDB cluster some-name-rs0' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster some-name-rs0 ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/conf/some-name-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/conf/some-name-rs0.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/conf/some-name-rs0.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + local LAST_OUT=/tmp/tmp.oOFTqYzuCc + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1915-2d829121"' + yq eval '.spec.upgradeOptions.apply="Never"' ++ mktemp + local LAST_ERR=/tmp/tmp.78LlXwN7w7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oOFTqYzuCc perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.78LlXwN7w7 + rm /tmp/tmp.oOFTqYzuCc /tmp/tmp.78LlXwN7w7 + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready........OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yVGQxiMeqH +++ mktemp ++ local LAST_ERR=/tmp/tmp.LlNRhDLhkx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yVGQxiMeqH ++ cat /tmp/tmp.LlNRhDLhkx ++ rm /tmp/tmp.yVGQxiMeqH /tmp/tmp.LlNRhDLhkx ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready........OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5N8JlXtVBL +++ mktemp ++ local LAST_ERR=/tmp/tmp.CZDSUltPLH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5N8JlXtVBL ++ cat /tmp/tmp.CZDSUltPLH ++ rm /tmp/tmp.5N8JlXtVBL /tmp/tmp.CZDSUltPLH ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + desc 'check if statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/some-name-rs0 + local resource=statefulset/some-name-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/statefulset_some-name-rs0.yml + local new_result=/tmp/tmp.bcF4VhRS4h/statefulset_some-name-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/statefulset_some-name-rs0-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs0 ++ mktemp + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("pitr-8397", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - + local LAST_OUT=/tmp/tmp.trfb825Sjr ++ mktemp + local LAST_ERR=/tmp/tmp.BrH1pggy2j + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.trfb825Sjr + cat /tmp/tmp.BrH1pggy2j + rm /tmp/tmp.trfb825Sjr /tmp/tmp.BrH1pggy2j + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.bcF4VhRS4h/statefulset_some-name-rs0.yml + version_gt 1.22 ++ echo '1.30 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.bcF4VhRS4h/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.bcF4VhRS4h/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/statefulset_some-name-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/statefulset_some-name-rs0.yml /tmp/tmp.bcF4VhRS4h/statefulset_some-name-rs0.yml + write_initial_data + desc 'create user myApp' + set +o xtrace ----------------------------------------------------------------------------------- create user myApp ----------------------------------------------------------------------------------- + run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.pitr-8397 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.pitr-8397 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OY8xHW2hiI +++ mktemp ++ local LAST_ERR=/tmp/tmp.YIQJwhS2lS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OY8xHW2hiI ++ cat /tmp/tmp.YIQJwhS2lS ++ rm /tmp/tmp.OY8xHW2hiI /tmp/tmp.YIQJwhS2lS ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ userAdmin:userAdmin123456@some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.lehgV2AvxP ++ mktemp + local LAST_ERR=/tmp/tmp.LuXLRfJnWV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lehgV2AvxP Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("c65cc6bb-5cb7-4e16-a694-b40350d8a9e2") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.LuXLRfJnWV + rm /tmp/tmp.lehgV2AvxP /tmp/tmp.LuXLRfJnWV + return 0 + sleep 2 + write_document + local cmp_postfix= + desc 'write initial data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write initial data, read from all ----------------------------------------------------------------------------------- + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pitr-8397 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.pitr-8397 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ihCEBoPjno +++ mktemp ++ local LAST_ERR=/tmp/tmp.wiCL3YAuDX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ihCEBoPjno ++ cat /tmp/tmp.wiCL3YAuDX ++ rm /tmp/tmp.ihCEBoPjno /tmp/tmp.wiCL3YAuDX ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.MxOAxyf6zZ ++ mktemp + local LAST_ERR=/tmp/tmp.eFAzCDRKcy + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MxOAxyf6zZ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("7b0d85fc-74dc-476f-941b-6830d52d9552") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.eFAzCDRKcy + rm /tmp/tmp.MxOAxyf6zZ /tmp/tmp.eFAzCDRKcy + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 '' + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:33:10+0000] running db.test.find() in myApp + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OZgZPkLx49 + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_ERR=/tmp/tmp.4COYjv3isS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OZgZPkLx49 ++ cat /tmp/tmp.4COYjv3isS ++ rm /tmp/tmp.OZgZPkLx49 /tmp/tmp.4COYjv3isS ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.e50ANRKCD9 ++ mktemp + local LAST_ERR=/tmp/tmp.z8dvNzu7f3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.e50ANRKCD9 + cat /tmp/tmp.z8dvNzu7f3 + rm /tmp/tmp.e50ANRKCD9 /tmp/tmp.z8dvNzu7f3 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find.json /tmp/tmp.bcF4VhRS4h/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 '' + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:33:12+0000] running db.test.find() in myApp + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NhpeEV7lQK +++ mktemp ++ local LAST_ERR=/tmp/tmp.SbIsDtXm38 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NhpeEV7lQK ++ cat /tmp/tmp.SbIsDtXm38 ++ rm /tmp/tmp.NhpeEV7lQK /tmp/tmp.SbIsDtXm38 ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Rj3LerW5Cf ++ mktemp + local LAST_ERR=/tmp/tmp.48gEt7m7Ul + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Rj3LerW5Cf + cat /tmp/tmp.48gEt7m7Ul + rm /tmp/tmp.Rj3LerW5Cf /tmp/tmp.48gEt7m7Ul + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find.json /tmp/tmp.bcF4VhRS4h/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 '' + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:33:14+0000] running db.test.find() in myApp + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + local driver=mongodb + local suffix=.svc.cluster.local + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BiKH7ztYnK +++ mktemp ++ local LAST_ERR=/tmp/tmp.G5MoV0qJJG ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BiKH7ztYnK ++ cat /tmp/tmp.G5MoV0qJJG ++ rm /tmp/tmp.BiKH7ztYnK /tmp/tmp.G5MoV0qJJG ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.J1HfaK0XHA ++ mktemp + local LAST_ERR=/tmp/tmp.CxFwEL8uR5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.J1HfaK0XHA + cat /tmp/tmp.CxFwEL8uR5 + rm /tmp/tmp.J1HfaK0XHA /tmp/tmp.CxFwEL8uR5 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find.json /tmp/tmp.bcF4VhRS4h/find + wait_backup_agent some-name-rs0-0 + local agent_pod=some-name-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-0...2025-05-21T10:32:57.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-1 + local agent_pod=some-name-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-1...2025-05-21T10:32:57.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-2 + local agent_pod=some-name-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-2...2025-05-21T10:33:03.000+0000 I listening for the commands + backup_name_minio=backup-minio + run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-8397 + local 'command=use myApp\n db.test2.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.pitr-8397 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mOUuSk06j7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.rR2F6EPqH7 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mOUuSk06j7 ++ cat /tmp/tmp.rR2F6EPqH7 ++ rm /tmp/tmp.mOUuSk06j7 /tmp/tmp.rR2F6EPqH7 ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.M0qKkkGxOR ++ mktemp + local LAST_ERR=/tmp/tmp.yf68YpTmQT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.M0qKkkGxOR Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("48b398ab-4cfd-4a05-a33c-9e33eb5b0e8c") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.yf68YpTmQT + rm /tmp/tmp.M0qKkkGxOR /tmp/tmp.yf68YpTmQT + return 0 + run_backup backup-minio 0 + local name=backup-minio + local idx=0 + desc 'run backup backup-minio-0' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-0 ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/conf/backup-minio.yml + /usr/bin/sed -e 's/name:/name: backup-minio-0/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.iTSa1tA68h ++ mktemp + local LAST_ERR=/tmp/tmp.ABTHX0Vw4K + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iTSa1tA68h perconaservermongodbbackup.psmdb.percona.com/backup-minio-0 created + cat /tmp/tmp.ABTHX0Vw4K + rm /tmp/tmp.iTSa1tA68h /tmp/tmp.ABTHX0Vw4K + return 0 + wait_backup backup-minio-0 + local backup_name=backup-minio-0 + local target_state=ready + set +o xtrace waiting for backup-minio-0 to reach ready state..... + write_document -2nd + local cmp_postfix=-2nd + desc 'write initial data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write initial data, read from all ----------------------------------------------------------------------------------- + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pitr-8397 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.pitr-8397 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PNsx5EKAI7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.0EPGlqSySP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PNsx5EKAI7 ++ cat /tmp/tmp.0EPGlqSySP ++ rm /tmp/tmp.PNsx5EKAI7 /tmp/tmp.0EPGlqSySP ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.NFwJzjUcuv ++ mktemp + local LAST_ERR=/tmp/tmp.YyTAg4bXxb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NFwJzjUcuv Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("df059375-da69-4bb5-a14c-53a505fdce03") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.YyTAg4bXxb + rm /tmp/tmp.NFwJzjUcuv /tmp/tmp.YyTAg4bXxb + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 -2nd + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:33:41+0000] running db.test.find() in myApp + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.o5tMBnj03q +++ mktemp ++ local LAST_ERR=/tmp/tmp.dUFwivWFG6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.o5tMBnj03q ++ cat /tmp/tmp.dUFwivWFG6 ++ rm /tmp/tmp.o5tMBnj03q /tmp/tmp.dUFwivWFG6 ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.t9qigolQfC ++ mktemp + local LAST_ERR=/tmp/tmp.7pNbjkNVgt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.t9qigolQfC + cat /tmp/tmp.7pNbjkNVgt + rm /tmp/tmp.t9qigolQfC /tmp/tmp.7pNbjkNVgt + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.bcF4VhRS4h/find-2nd + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 -2nd + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:33:43+0000] running db.test.find() in myApp + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wa50J4wQxz +++ mktemp ++ local LAST_ERR=/tmp/tmp.berX3korBp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wa50J4wQxz ++ cat /tmp/tmp.berX3korBp ++ rm /tmp/tmp.wa50J4wQxz /tmp/tmp.berX3korBp ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.L0BImoW9M1 ++ mktemp + local LAST_ERR=/tmp/tmp.DkkzvMSrF0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.L0BImoW9M1 + cat /tmp/tmp.DkkzvMSrF0 + rm /tmp/tmp.L0BImoW9M1 /tmp/tmp.DkkzvMSrF0 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.bcF4VhRS4h/find-2nd + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 -2nd + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:33:46+0000] running db.test.find() in myApp + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.znQxTSI5aD +++ mktemp ++ local LAST_ERR=/tmp/tmp.hLZBOrhPIe ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.znQxTSI5aD ++ cat /tmp/tmp.hLZBOrhPIe ++ rm /tmp/tmp.znQxTSI5aD /tmp/tmp.hLZBOrhPIe ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.6sXbFgaA43 ++ mktemp + local LAST_ERR=/tmp/tmp.sjpZqinwjq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6sXbFgaA43 + cat /tmp/tmp.sjpZqinwjq + rm /tmp/tmp.6sXbFgaA43 /tmp/tmp.sjpZqinwjq + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.bcF4VhRS4h/find-2nd + sleep 2 ++ run_mongo 'new Date().toISOString()' myApp:myPass@some-name-rs0.pitr-8397 mongodb '' --quiet ++ local 'command=new Date().toISOString()' ++ local uri=myApp:myPass@some-name-rs0.pitr-8397 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ tr T ' ' ++ cut -c1-19 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match' +++ local LAST_OUT=/tmp/tmp.80DYVjx87K ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cefvv0Y0wn +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.80DYVjx87K +++ cat /tmp/tmp.cefvv0Y0wn +++ rm /tmp/tmp.80DYVjx87K /tmp/tmp.cefvv0Y0wn +++ return 0 ++ local client_container=psmdb-client-66f577db5f-bpkrq ++ local mongo_flag=--quiet ++ [[ myApp:myPass@some-name-rs0.pitr-8397 == *cfg* ]] ++ replica_set=rs0 ++ kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8ydQlbNlrW +++ mktemp ++ local LAST_ERR=/tmp/tmp.yfscM7XfWg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8ydQlbNlrW ++ cat /tmp/tmp.yfscM7XfWg ++ rm /tmp/tmp.8ydQlbNlrW /tmp/tmp.yfscM7XfWg ++ return 0 + time_now='2025-05-21 10:33:53' + check_recovery backup-minio-0 date '2025-05-21 10:33:53' -2nd '' some-name test2 + local backup_name=backup-minio-0 + local restore_type=date + local 'restore_date=2025-05-21 10:33:53' + local cmp_postfix=-2nd + local backupSource= + local cluster_name=some-name + local selective_collection=test2 + desc 'write more data before restore by date' + set +o xtrace ----------------------------------------------------------------------------------- write more data before restore by date ----------------------------------------------------------------------------------- + sleep 60 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-8397 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.pitr-8397 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wtyG9fsKo3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.rK2Z3Lh5lF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wtyG9fsKo3 ++ cat /tmp/tmp.rK2Z3Lh5lF ++ rm /tmp/tmp.wtyG9fsKo3 /tmp/tmp.rK2Z3Lh5lF ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.ixcGLEiuOj ++ mktemp + local LAST_ERR=/tmp/tmp.j4RMW7fc2a + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ixcGLEiuOj Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("ecfffa64-02c3-4f8f-83c3-87c43f0f1758") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.j4RMW7fc2a + rm /tmp/tmp.ixcGLEiuOj /tmp/tmp.j4RMW7fc2a + return 0 + '[' -n test2 ']' + run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.pitr-8397 + local 'command=use myApp\n db.test2.drop()' + local uri=myApp:myPass@some-name-rs0.pitr-8397 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dOXLQPnRcM +++ mktemp ++ local LAST_ERR=/tmp/tmp.DG7LjfDR1F ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dOXLQPnRcM ++ cat /tmp/tmp.DG7LjfDR1F ++ rm /tmp/tmp.dOXLQPnRcM /tmp/tmp.DG7LjfDR1F ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.a6YNJ04asv ++ mktemp + local LAST_ERR=/tmp/tmp.zlZoFLq7CI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.a6YNJ04asv Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("284a6158-b157-4423-94a2-dfea589cf1a6") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.zlZoFLq7CI + rm /tmp/tmp.a6YNJ04asv /tmp/tmp.zlZoFLq7CI + return 0 + desc 'waiting for chunks to be uploaded' + set +o xtrace ----------------------------------------------------------------------------------- waiting for chunks to be uploaded ----------------------------------------------------------------------------------- + sleep 120 + desc 'check restore by date' + set +o xtrace ----------------------------------------------------------------------------------- check restore by date ----------------------------------------------------------------------------------- + '[' -z '' ']' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/conf/restore.yml + /usr/bin/sed -e 's/backupName:/backupName: backup-minio-0/' + '[' -n test2 ']' + yq eval '.spec.selective = {"namespaces": ["myApp.test"], "withUsersAndRoles": true}' + /usr/bin/sed -e /backupSource/,+2d + /usr/bin/sed -e 's/name:/name: restore-backup-minio-0/' + kubectl_bin apply -f - + '[' -z '2025-05-21 10:33:53' ']' + /usr/bin/sed -e 's/date:/date: 2025-05-21 10:33:53/' ++ mktemp + local LAST_OUT=/tmp/tmp.2zysZOxDBW ++ mktemp + local LAST_ERR=/tmp/tmp.juhwcPNHCO + local exit_status=0 + local timeout=4 + /usr/bin/sed -e 's/pitrType:/type: date/' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2zysZOxDBW perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-0 created + cat /tmp/tmp.juhwcPNHCO + rm /tmp/tmp.2zysZOxDBW /tmp/tmp.juhwcPNHCO + return 0 + wait_restore backup-minio-0 some-name + local backup_name=backup-minio-0 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-0 object to be createdOK Waiting psmdb-restore/restore-backup-minio-0 to reach state "ready" OK + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8JYLwkDQ89 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4Oe6YaUDF1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8JYLwkDQ89 ++ cat /tmp/tmp.4Oe6YaUDF1 ++ rm /tmp/tmp.8JYLwkDQ89 /tmp/tmp.4Oe6YaUDF1 ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo + echo + set -o xtrace + '[' -n test2 ']' ++ collection_exists test2 ./e2e-tests/pitr/run: line 100: collection_exists: command not found + [[ '' == \t\r\u\e ]] + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 -2nd + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:37:25+0000] running db.test.find() in myApp + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + local driver=mongodb + local suffix=.svc.cluster.local + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f1SzrCeFpI +++ mktemp ++ local LAST_ERR=/tmp/tmp.6WIz7rNBL6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.f1SzrCeFpI ++ cat /tmp/tmp.6WIz7rNBL6 ++ rm /tmp/tmp.f1SzrCeFpI /tmp/tmp.6WIz7rNBL6 ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.x6MfvDjCbB ++ mktemp + local LAST_ERR=/tmp/tmp.a0PoisT5T7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.x6MfvDjCbB + cat /tmp/tmp.a0PoisT5T7 + rm /tmp/tmp.x6MfvDjCbB /tmp/tmp.a0PoisT5T7 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.bcF4VhRS4h/find-2nd + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 -2nd + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:37:27+0000] running db.test.find() in myApp + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qyemyuII82 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7xBuGIGLHS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qyemyuII82 ++ cat /tmp/tmp.7xBuGIGLHS ++ rm /tmp/tmp.qyemyuII82 /tmp/tmp.7xBuGIGLHS ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Hvhvx6s5SN ++ mktemp + local LAST_ERR=/tmp/tmp.t6Vh1MXoMR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Hvhvx6s5SN + cat /tmp/tmp.t6Vh1MXoMR + rm /tmp/tmp.Hvhvx6s5SN /tmp/tmp.t6Vh1MXoMR + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.bcF4VhRS4h/find-2nd + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 -2nd + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:37:30+0000] running db.test.find() in myApp + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xOVG1B3EpW +++ mktemp ++ local LAST_ERR=/tmp/tmp.2HrXb6o7KG ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xOVG1B3EpW ++ cat /tmp/tmp.2HrXb6o7KG ++ rm /tmp/tmp.xOVG1B3EpW /tmp/tmp.2HrXb6o7KG ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.IrLeFSz5cq ++ mktemp + local LAST_ERR=/tmp/tmp.9SoWTdNTri + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IrLeFSz5cq + cat /tmp/tmp.9SoWTdNTri + rm /tmp/tmp.IrLeFSz5cq /tmp/tmp.9SoWTdNTri + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find-2nd.json /tmp/tmp.bcF4VhRS4h/find-2nd + run_backup backup-minio 1 + local name=backup-minio + local idx=1 + desc 'run backup backup-minio-1' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-1 ----------------------------------------------------------------------------------- + /usr/bin/sed -e 's/name:/name: backup-minio-1/' + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/conf/backup-minio.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xcDNmeoyRa ++ mktemp + local LAST_ERR=/tmp/tmp.mtN6jYDksZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xcDNmeoyRa perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created + cat /tmp/tmp.mtN6jYDksZ + rm /tmp/tmp.xcDNmeoyRa /tmp/tmp.mtN6jYDksZ + return 0 + wait_backup backup-minio-1 + local backup_name=backup-minio-1 + local target_state=ready + set +o xtrace waiting for backup-minio-1 to reach ready state..... + compare_latest_restorable_time some-name-rs0 backup-minio-1 + local cluster=some-name-rs0 + local backup_name=backup-minio-1 + local latest_restorable_time + local backup_time ++ get_latest_restorable_time some-name-rs0 ++ local cluster=some-name-rs0 ++ local first_timestamp ++ local second_timestamp ++ local retry=0 ++ [[ '' != '' ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ChY9QsqWs3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hjMcAplBTw +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ChY9QsqWs3 +++ cat /tmp/tmp.hjMcAplBTw +++ rm /tmp/tmp.ChY9QsqWs3 /tmp/tmp.hjMcAplBTw +++ return 0 ++ first_timestamp=1747823825 ++ sleep 5 ++ [[ 1747823825 != '' ]] ++ [[ 1747823825 != \n\u\l\l ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.anPdXHf5sO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7L6fnATW6c +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.anPdXHf5sO +++ cat /tmp/tmp.7L6fnATW6c +++ rm /tmp/tmp.anPdXHf5sO /tmp/tmp.7L6fnATW6c +++ return 0 ++ second_timestamp=1747823825 ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 1747823825 != '' ]] ++ [[ 1747823825 != \n\u\l\l ]] ++ [[ 1747823825 == 1747823825 ]] ++ /usr/bin/date -u -d @1747823825 +%Y-%m-%dT%H:%M:%SZ + latest_restorable_time=2025-05-21T10:37:05Z ++ get_latest_restorable_time_from_backup_object backup-minio-1 ++ local backup_name=backup-minio-1 ++ local latestRestorableTime ++ local retry=0 ++ [[ '' != '' ]] ++ sleep 5 +++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.j8uAmzK42K ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YNxSNSpbXs +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.j8uAmzK42K +++ cat /tmp/tmp.YNxSNSpbXs +++ rm /tmp/tmp.j8uAmzK42K /tmp/tmp.YNxSNSpbXs +++ return 0 ++ latestRestorableTime=2025-05-21T10:37:05Z ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 2025-05-21T10:37:05Z != '' ]] ++ [[ 2025-05-21T10:37:05Z != \n\u\l\l ]] ++ echo 2025-05-21T10:37:05Z + backup_time=2025-05-21T10:37:05Z + [[ 2025-05-21T10:37:05Z != \2\0\2\5\-\0\5\-\2\1\T\1\0\:\3\7\:\0\5\Z ]] + check_recovery backup-minio-1 latest '' -3rd '' some-name + local backup_name=backup-minio-1 + local restore_type=latest + local restore_date= + local cmp_postfix=-3rd + local backupSource= + local cluster_name=some-name + local selective_collection= + desc 'write more data before restore by latest' + set +o xtrace ----------------------------------------------------------------------------------- write more data before restore by latest ----------------------------------------------------------------------------------- + sleep 60 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-8397 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.pitr-8397 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ff4tiuO69N +++ mktemp ++ local LAST_ERR=/tmp/tmp.tMgOBzdb8b ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ff4tiuO69N ++ cat /tmp/tmp.tMgOBzdb8b ++ rm /tmp/tmp.Ff4tiuO69N /tmp/tmp.tMgOBzdb8b ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.92jFJfRmG4 ++ mktemp + local LAST_ERR=/tmp/tmp.46xkxf5kbQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.92jFJfRmG4 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("22975f11-b6a7-439e-8224-3df03ae81924") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.46xkxf5kbQ + rm /tmp/tmp.92jFJfRmG4 /tmp/tmp.46xkxf5kbQ + return 0 + '[' -n '' ']' + desc 'waiting for chunks to be uploaded' + set +o xtrace ----------------------------------------------------------------------------------- waiting for chunks to be uploaded ----------------------------------------------------------------------------------- + sleep 120 + desc 'check restore by latest' + set +o xtrace ----------------------------------------------------------------------------------- check restore by latest ----------------------------------------------------------------------------------- + '[' -z '' ']' + /usr/bin/sed -e 's/name:/name: restore-backup-minio-1/' + /usr/bin/sed -e 's/backupName:/backupName: backup-minio-1/' + /usr/bin/sed -e /backupSource/,+2d + '[' -n '' ']' + yq + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/conf/restore.yml + /usr/bin/sed -e 's/pitrType:/type: latest/' + '[' -z '' ']' + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e /date:/d + local LAST_OUT=/tmp/tmp.fx9BjkqFFB ++ mktemp + local LAST_ERR=/tmp/tmp.QeF6XMFjIR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fx9BjkqFFB perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created + cat /tmp/tmp.QeF6XMFjIR + rm /tmp/tmp.fx9BjkqFFB /tmp/tmp.QeF6XMFjIR + return 0 + wait_restore backup-minio-1 some-name + local backup_name=backup-minio-1 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-1 object to be createdOK Waiting psmdb-restore/restore-backup-minio-1 to reach state "ready" OK + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yAEY0DYrSY +++ mktemp ++ local LAST_ERR=/tmp/tmp.N6qDlNssF5 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yAEY0DYrSY ++ cat /tmp/tmp.N6qDlNssF5 ++ rm /tmp/tmp.yAEY0DYrSY /tmp/tmp.N6qDlNssF5 ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo + echo + set -o xtrace + '[' -n '' ']' + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 -3rd + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 + local postfix=-3rd + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:41:33+0000] running db.test.find() in myApp + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8rghFsE5j6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.pbqqRA0p1I ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8rghFsE5j6 ++ cat /tmp/tmp.pbqqRA0p1I ++ rm /tmp/tmp.8rghFsE5j6 /tmp/tmp.pbqqRA0p1I ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.ecAPCOcbSg ++ mktemp + local LAST_ERR=/tmp/tmp.dDnesKfHbW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ecAPCOcbSg + cat /tmp/tmp.dDnesKfHbW + rm /tmp/tmp.ecAPCOcbSg /tmp/tmp.dDnesKfHbW + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find-3rd.json /tmp/tmp.bcF4VhRS4h/find-3rd + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 -3rd + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 + local postfix=-3rd + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:41:37+0000] running db.test.find() in myApp + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lITjd1OhcQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.9aC4ScN5Yc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lITjd1OhcQ ++ cat /tmp/tmp.9aC4ScN5Yc ++ rm /tmp/tmp.lITjd1OhcQ /tmp/tmp.9aC4ScN5Yc ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.SWOYrpIsfJ ++ mktemp + local LAST_ERR=/tmp/tmp.bNA5wFsJR6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SWOYrpIsfJ + cat /tmp/tmp.bNA5wFsJR6 + rm /tmp/tmp.SWOYrpIsfJ /tmp/tmp.bNA5wFsJR6 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find-3rd.json /tmp/tmp.bcF4VhRS4h/find-3rd + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 -3rd + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 + local postfix=-3rd + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:41:40+0000] running db.test.find() in myApp + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 mongodb '' + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ local LAST_OUT=/tmp/tmp.P656BiioZB +++ mktemp ++ local LAST_ERR=/tmp/tmp.2NOa1CCvYA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.P656BiioZB ++ cat /tmp/tmp.2NOa1CCvYA ++ rm /tmp/tmp.P656BiioZB /tmp/tmp.2NOa1CCvYA ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.nE6jCmSq1M ++ mktemp + local LAST_ERR=/tmp/tmp.ou9Kb4p1Sz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nE6jCmSq1M + cat /tmp/tmp.ou9Kb4p1Sz + rm /tmp/tmp.nE6jCmSq1M /tmp/tmp.ou9Kb4p1Sz + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find-3rd.json /tmp/tmp.bcF4VhRS4h/find-3rd + desc 'restore pitr using backupSource' + set +o xtrace ----------------------------------------------------------------------------------- restore pitr using backupSource ----------------------------------------------------------------------------------- + reset_collection + desc 'reset data' + set +o xtrace ----------------------------------------------------------------------------------- reset data ----------------------------------------------------------------------------------- + run_mongo 'use myApp\n db.test.remove({})' myApp:myPass@some-name-rs0.pitr-8397 + local 'command=use myApp\n db.test.remove({})' + local uri=myApp:myPass@some-name-rs0.pitr-8397 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qYJqibr2ys +++ mktemp ++ local LAST_ERR=/tmp/tmp.NXD5ZWafQW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qYJqibr2ys ++ cat /tmp/tmp.NXD5ZWafQW ++ rm /tmp/tmp.qYJqibr2ys /tmp/tmp.NXD5ZWafQW ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.ny2GwKzNqY ++ mktemp + local LAST_ERR=/tmp/tmp.VoivFB2RYJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ny2GwKzNqY Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("8aeaba73-2b46-4fb3-aee0-ad712a356940") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nRemoved" : 3 }) bye + cat /tmp/tmp.VoivFB2RYJ + rm /tmp/tmp.ny2GwKzNqY /tmp/tmp.VoivFB2RYJ + return 0 + sleep 2 + write_document + local cmp_postfix= + desc 'write initial data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write initial data, read from all ----------------------------------------------------------------------------------- + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pitr-8397 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.pitr-8397 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cjBQdszMSb +++ mktemp ++ local LAST_ERR=/tmp/tmp.b21jzhEdhj ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cjBQdszMSb ++ cat /tmp/tmp.b21jzhEdhj ++ rm /tmp/tmp.cjBQdszMSb /tmp/tmp.b21jzhEdhj ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.QK8G204p7C ++ mktemp + local LAST_ERR=/tmp/tmp.yjkPzKxkcH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QK8G204p7C Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("040bd7d5-e68a-45a5-ad5c-79dd902a5951") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.yjkPzKxkcH + rm /tmp/tmp.QK8G204p7C /tmp/tmp.yjkPzKxkcH + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 '' + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:41:50+0000] running db.test.find() in myApp + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Xs1zGOrdQA +++ mktemp ++ local LAST_ERR=/tmp/tmp.o2HV0iJpWp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Xs1zGOrdQA ++ cat /tmp/tmp.o2HV0iJpWp ++ rm /tmp/tmp.Xs1zGOrdQA /tmp/tmp.o2HV0iJpWp ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.j2alE9iHUe ++ mktemp + local LAST_ERR=/tmp/tmp.lWIONsmBSW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.j2alE9iHUe + cat /tmp/tmp.lWIONsmBSW + rm /tmp/tmp.j2alE9iHUe /tmp/tmp.lWIONsmBSW + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find.json /tmp/tmp.bcF4VhRS4h/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 '' + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:41:54+0000] running db.test.find() in myApp + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ local LAST_OUT=/tmp/tmp.tbHWbKYiQG +++ mktemp ++ local LAST_ERR=/tmp/tmp.6bk8PX5EN4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tbHWbKYiQG ++ cat /tmp/tmp.6bk8PX5EN4 ++ rm /tmp/tmp.tbHWbKYiQG /tmp/tmp.6bk8PX5EN4 ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Dbgbj2zC1r ++ mktemp + local LAST_ERR=/tmp/tmp.0Bnw0R4Yst + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Dbgbj2zC1r + cat /tmp/tmp.0Bnw0R4Yst + rm /tmp/tmp.Dbgbj2zC1r /tmp/tmp.0Bnw0R4Yst + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find.json /tmp/tmp.bcF4VhRS4h/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 '' + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:41:57+0000] running db.test.find() in myApp + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ local LAST_OUT=/tmp/tmp.nzq0PbSs7R +++ mktemp ++ local LAST_ERR=/tmp/tmp.2Uk8NAxwiC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nzq0PbSs7R ++ cat /tmp/tmp.2Uk8NAxwiC ++ rm /tmp/tmp.nzq0PbSs7R /tmp/tmp.2Uk8NAxwiC ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.EoGEna0an3 ++ mktemp + local LAST_ERR=/tmp/tmp.h3JrLcSSFl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EoGEna0an3 + cat /tmp/tmp.h3JrLcSSFl + rm /tmp/tmp.EoGEna0an3 /tmp/tmp.h3JrLcSSFl + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find.json /tmp/tmp.bcF4VhRS4h/find + run_backup backup-minio 2 + local name=backup-minio + local idx=2 + desc 'run backup backup-minio-2' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-2 ----------------------------------------------------------------------------------- + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.lnP7XMfeMy ++ mktemp + local LAST_ERR=/tmp/tmp.pBUQYrjKBC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/conf/backup-minio.yml + /usr/bin/sed -e 's/name:/name: backup-minio-2/' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lnP7XMfeMy perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created + cat /tmp/tmp.pBUQYrjKBC + rm /tmp/tmp.lnP7XMfeMy /tmp/tmp.pBUQYrjKBC + return 0 + wait_backup backup-minio-2 + local backup_name=backup-minio-2 + local target_state=ready + set +o xtrace waiting for backup-minio-2 to reach ready state...... + compare_latest_restorable_time some-name-rs0 backup-minio-2 + local cluster=some-name-rs0 + local backup_name=backup-minio-2 + local latest_restorable_time + local backup_time ++ get_latest_restorable_time some-name-rs0 ++ local cluster=some-name-rs0 ++ local first_timestamp ++ local second_timestamp ++ local retry=0 ++ [[ '' != '' ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3qBGXDP8H4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6ZWqrS3BL1 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.3qBGXDP8H4 +++ cat /tmp/tmp.6ZWqrS3BL1 +++ rm /tmp/tmp.3qBGXDP8H4 /tmp/tmp.6ZWqrS3BL1 +++ return 0 ++ first_timestamp=1747824070 ++ sleep 5 ++ [[ 1747824070 != '' ]] ++ [[ 1747824070 != \n\u\l\l ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XfsJlZ2IFm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tr5fA0SUF4 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.XfsJlZ2IFm +++ cat /tmp/tmp.tr5fA0SUF4 +++ rm /tmp/tmp.XfsJlZ2IFm /tmp/tmp.tr5fA0SUF4 +++ return 0 ++ second_timestamp=1747824070 ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 1747824070 != '' ]] ++ [[ 1747824070 != \n\u\l\l ]] ++ [[ 1747824070 == 1747824070 ]] ++ /usr/bin/date -u -d @1747824070 +%Y-%m-%dT%H:%M:%SZ + latest_restorable_time=2025-05-21T10:41:10Z ++ get_latest_restorable_time_from_backup_object backup-minio-2 ++ local backup_name=backup-minio-2 ++ local latestRestorableTime ++ local retry=0 ++ [[ '' != '' ]] ++ sleep 5 +++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EyTbudVsrS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.noBYDU7VNw +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.EyTbudVsrS +++ cat /tmp/tmp.noBYDU7VNw +++ rm /tmp/tmp.EyTbudVsrS /tmp/tmp.noBYDU7VNw +++ return 0 ++ latestRestorableTime=2025-05-21T10:41:10Z ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 2025-05-21T10:41:10Z != '' ]] ++ [[ 2025-05-21T10:41:10Z != \n\u\l\l ]] ++ echo 2025-05-21T10:41:10Z + backup_time=2025-05-21T10:41:10Z + [[ 2025-05-21T10:41:10Z != \2\0\2\5\-\0\5\-\2\1\T\1\0\:\4\1\:\1\0\Z ]] ++ cut -c1-19 ++ run_mongo 'new Date().toISOString()' myApp:myPass@some-name-rs0.pitr-8397 mongodb '' --quiet ++ local 'command=new Date().toISOString()' ++ local uri=myApp:myPass@some-name-rs0.pitr-8397 ++ local driver=mongodb ++ local suffix=.svc.cluster.local +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match' ++ tr T ' ' +++ local LAST_OUT=/tmp/tmp.oGf218LtUl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0s5gom71CU +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.oGf218LtUl +++ cat /tmp/tmp.0s5gom71CU +++ rm /tmp/tmp.oGf218LtUl /tmp/tmp.0s5gom71CU +++ return 0 ++ local client_container=psmdb-client-66f577db5f-bpkrq ++ local mongo_flag=--quiet ++ [[ myApp:myPass@some-name-rs0.pitr-8397 == *cfg* ]] ++ replica_set=rs0 ++ kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tMKCTM5LHe +++ mktemp ++ local LAST_ERR=/tmp/tmp.iTDObzNuYZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''new Date().toISOString()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tMKCTM5LHe ++ cat /tmp/tmp.iTDObzNuYZ ++ rm /tmp/tmp.tMKCTM5LHe /tmp/tmp.iTDObzNuYZ ++ return 0 + time_now='2025-05-21 10:42:32' + check_recovery backup-minio-2 date '2025-05-21 10:42:32' '' backupSource some-name + local backup_name=backup-minio-2 + local restore_type=date + local 'restore_date=2025-05-21 10:42:32' + local cmp_postfix= + local backupSource=backupSource + local cluster_name=some-name + local selective_collection= + desc 'write more data before restore by date' + set +o xtrace ----------------------------------------------------------------------------------- write more data before restore by date ----------------------------------------------------------------------------------- + sleep 60 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.pitr-8397 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.pitr-8397 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yaxnqbfttc +++ mktemp ++ local LAST_ERR=/tmp/tmp.S2aw1LEtpN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yaxnqbfttc ++ cat /tmp/tmp.S2aw1LEtpN ++ rm /tmp/tmp.yaxnqbfttc /tmp/tmp.S2aw1LEtpN ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.ymuXx2t5Eg ++ mktemp + local LAST_ERR=/tmp/tmp.EoMowbLH5E + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ymuXx2t5Eg Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("0927f06a-3d1c-4d21-bcc4-2a621f26e95e") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.EoMowbLH5E + rm /tmp/tmp.ymuXx2t5Eg /tmp/tmp.EoMowbLH5E + return 0 + '[' -n '' ']' + desc 'waiting for chunks to be uploaded' + set +o xtrace ----------------------------------------------------------------------------------- waiting for chunks to be uploaded ----------------------------------------------------------------------------------- + sleep 120 + desc 'check restore by date' + set +o xtrace ----------------------------------------------------------------------------------- check restore by date ----------------------------------------------------------------------------------- + '[' -z backupSource ']' ++ get_backup_dest backup-minio-2 ++ local backup_name=backup-minio-2 ++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}' ++ sed 's|azure://||' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FziiWNUCZo +++ mktemp ++ local LAST_ERR=/tmp/tmp.2YHrb9J3D6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FziiWNUCZo ++ cat /tmp/tmp.2YHrb9J3D6 ++ rm /tmp/tmp.FziiWNUCZo /tmp/tmp.2YHrb9J3D6 ++ return 0 + backup_dest=operator-testing/2025-05-21T10:42:03Z + /usr/bin/sed -e 's/name:/name: restore-backup-minio-2/' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/conf/restore.yml + '[' -n '' ']' + kubectl_bin apply -f - + yq ++ mktemp + local LAST_OUT=/tmp/tmp.rGcztG5ILa ++ mktemp + /usr/bin/sed -e 's|BACKUP-NAME|operator-testing/2025-05-21T10:42:03Z|' + local LAST_ERR=/tmp/tmp.N4TFYxaF5J + local exit_status=0 + local timeout=4 ++ seq 0 2 + /usr/bin/sed -e /backupName/d + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's/pitrType:/type: date/' + '[' -z '2025-05-21 10:42:32' ']' + /usr/bin/sed -e 's/date:/date: 2025-05-21 10:42:32/' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rGcztG5ILa perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2 created + cat /tmp/tmp.N4TFYxaF5J + rm /tmp/tmp.rGcztG5ILa /tmp/tmp.N4TFYxaF5J + return 0 + wait_restore backup-minio-2 some-name + local backup_name=backup-minio-2 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-2 object to be createdOK Waiting psmdb-restore/restore-backup-minio-2 to reach state "ready" OK + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YuMWOKGxv4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.geCouYqjQV ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YuMWOKGxv4 ++ cat /tmp/tmp.geCouYqjQV ++ rm /tmp/tmp.YuMWOKGxv4 /tmp/tmp.geCouYqjQV ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo + echo + set -o xtrace + '[' -n '' ']' + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 '' + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:46:10+0000] running db.test.find() in myApp + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.I10Rm7RZIp +++ mktemp ++ local LAST_ERR=/tmp/tmp.emVaDDDKXB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.I10Rm7RZIp ++ cat /tmp/tmp.emVaDDDKXB ++ rm /tmp/tmp.I10Rm7RZIp /tmp/tmp.emVaDDDKXB ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.A0fhFzk2bR ++ mktemp + local LAST_ERR=/tmp/tmp.aBL1oMGkxL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.A0fhFzk2bR + cat /tmp/tmp.aBL1oMGkxL + rm /tmp/tmp.A0fhFzk2bR /tmp/tmp.aBL1oMGkxL + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find.json /tmp/tmp.bcF4VhRS4h/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 '' + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:46:12+0000] running db.test.find() in myApp + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.i68U3t3Sd6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.kGtPUFGlr2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.i68U3t3Sd6 ++ cat /tmp/tmp.kGtPUFGlr2 ++ rm /tmp/tmp.i68U3t3Sd6 /tmp/tmp.kGtPUFGlr2 ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.r5p58IFcIw ++ mktemp + local LAST_ERR=/tmp/tmp.30r3ycFeqW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.r5p58IFcIw + cat /tmp/tmp.30r3ycFeqW + rm /tmp/tmp.r5p58IFcIw /tmp/tmp.30r3ycFeqW + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find.json /tmp/tmp.bcF4VhRS4h/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 '' + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local 'full_command=db.test.find()' + [[ ! -z '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-05-21T10:46:15+0000] running db.test.find() in myApp + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lNaESzAQ0p +++ mktemp ++ local LAST_ERR=/tmp/tmp.hQsaHH3vsc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lNaESzAQ0p ++ cat /tmp/tmp.hQsaHH3vsc ++ rm /tmp/tmp.lNaESzAQ0p /tmp/tmp.hQsaHH3vsc ++ return 0 + local client_container=psmdb-client-66f577db5f-bpkrq + local mongo_flag= + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.LuxROtgvdi ++ mktemp + local LAST_ERR=/tmp/tmp.iBEeolTrly + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-bpkrq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LuxROtgvdi + cat /tmp/tmp.iBEeolTrly + rm /tmp/tmp.LuxROtgvdi /tmp/tmp.iBEeolTrly + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/compare/find.json /tmp/tmp.bcF4VhRS4h/find + run_backup backup-minio 3 + local name=backup-minio + local idx=3 + desc 'run backup backup-minio-3' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-3 ----------------------------------------------------------------------------------- + /usr/bin/sed -e 's/name:/name: backup-minio-3/' + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr/conf/backup-minio.yml ++ mktemp + local LAST_OUT=/tmp/tmp.doFNglF35b ++ mktemp + local LAST_ERR=/tmp/tmp.n6tn4uRWxo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.doFNglF35b perconaservermongodbbackup.psmdb.percona.com/backup-minio-3 created + cat /tmp/tmp.n6tn4uRWxo + rm /tmp/tmp.doFNglF35b /tmp/tmp.n6tn4uRWxo + return 0 + wait_backup backup-minio-3 + local backup_name=backup-minio-3 + local target_state=ready + set +o xtrace waiting for backup-minio-3 to reach ready state.........................................................2025-05-21T10:42:18.534Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-2", "reconcileID": "847f5e65-bdac-4071-84c9-5c36f2c42c9d"} 2025-05-21T10:45:39.812Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "93ca8a23-60cf-4521-a5da-f0ff060c8850", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:45:39.954Z INFO Waiting for PITR to be disabled. {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "93ca8a23-60cf-4521-a5da-f0ff060c8850"} 2025-05-21T10:45:39.955Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "93ca8a23-60cf-4521-a5da-f0ff060c8850", "previous": "", "current": "waiting"} 2025-05-21T10:45:40.532Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "6ca3d93d-674b-455a-be85-2b32202758bf", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:45:40.584Z INFO Waiting for PITR to be disabled. {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "6ca3d93d-674b-455a-be85-2b32202758bf"} 2025-05-21T10:45:45.533Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "f005b8f6-ba9b-4123-b163-9371fca2da61", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:45:45.587Z INFO Waiting for PITR to be disabled. {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "f005b8f6-ba9b-4123-b163-9371fca2da61"} 2025-05-21T10:45:50.589Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "6a42de79-ce50-4007-b71c-cc4eb5e52484", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:45:50.646Z INFO Starting restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "6a42de79-ce50-4007-b71c-cc4eb5e52484", "backup": "2025-05-21T10:42:03Z"} 2025-05-21T10:45:50.646Z INFO Starting logical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "6a42de79-ce50-4007-b71c-cc4eb5e52484", "backup": "2025-05-21T10:42:03Z"} 2025-05-21T10:45:50.649Z INFO Sending restore command {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "6a42de79-ce50-4007-b71c-cc4eb5e52484", "restoreCmd": "name: 2025-05-21T10:45:50.646898763Z, snapshot: 2025-05-21T10:42:03Z point-in-time: <1747824152,0>"} 2025-05-21T10:45:50.656Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "6a42de79-ce50-4007-b71c-cc4eb5e52484", "previous": "waiting", "current": "requested"} 2025-05-21T10:45:51.233Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "036851d8-9e89-45ee-ad5c-4a95deb3586e", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:45:51.282Z INFO Waiting for restore metadata {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "036851d8-9e89-45ee-ad5c-4a95deb3586e", "pbmName": "2025-05-21T10:45:50.646898763Z", "restore": "restore-backup-minio-2", "backup": ""} 2025-05-21T10:45:56.234Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "beb0b202-e711-46d7-88dd-99bd18664a9e", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:45:56.279Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "beb0b202-e711-46d7-88dd-99bd18664a9e", "previous": "requested", "current": "running"} 2025-05-21T10:45:56.833Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "998f272a-7f36-4a90-a1d2-9a5fd34bca46", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:01.834Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "2ac9cd3e-ba47-4a2a-9e48-26dbc0b00c94", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:01.892Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "restore-backup-minio-2", "reconcileID": "2ac9cd3e-ba47-4a2a-9e48-26dbc0b00c94", "previous": "running", "current": "ready"} 2025-05-21T10:46:06.606Z INFO PBM updating latest restorable time {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "some-name", "reconcileID": "2cc221bb-de62-41d8-9c6f-998edcc37f0f", "backup": "backup-minio-2", "latestRestorableTime": "2025-05-21 10:45:50 +0000 UTC"} 2025-05-21T10:46:06.656Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-2", "reconcileID": "eb3a6ed1-ce4c-44b3-9c37-085d6076c4ed"} 2025-05-21T10:46:06.656Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-2", "reconcileID": "eb3a6ed1-ce4c-44b3-9c37-085d6076c4ed"} 2025-05-21T10:46:18.685Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14da493a-308c-4382-9852-5b69669ca857"} 2025-05-21T10:46:18.736Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14da493a-308c-4382-9852-5b69669ca857", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:18.736Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14da493a-308c-4382-9852-5b69669ca857", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:46:18.783Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14da493a-308c-4382-9852-5b69669ca857"} 2025-05-21T10:46:18.955Z DEBUG Added finalizer {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14da493a-308c-4382-9852-5b69669ca857", "finalizer": "internal.percona.com/release-lock"} 2025-05-21T10:46:18.955Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14da493a-308c-4382-9852-5b69669ca857", "backup": "backup-minio-3", "storage": "minio"} 2025-05-21T10:46:18.955Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14da493a-308c-4382-9852-5b69669ca857", "backup": "backup-minio-3", "storage": "minio", "backupCmd": "backup [name: 2025-05-21T10:46:18Z, compression: gzip (level: default)] ", "profile": ""} 2025-05-21T10:46:18.963Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14da493a-308c-4382-9852-5b69669ca857", "previous": "", "current": "requested"} 2025-05-21T10:46:19.012Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14da493a-308c-4382-9852-5b69669ca857"} 2025-05-21T10:46:19.012Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e0f9f25f-180c-4ea3-8349-18b8d33e56c1"} 2025-05-21T10:46:19.058Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e0f9f25f-180c-4ea3-8349-18b8d33e56c1", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:19.058Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e0f9f25f-180c-4ea3-8349-18b8d33e56c1", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:46:19.103Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e0f9f25f-180c-4ea3-8349-18b8d33e56c1"} 2025-05-21T10:46:19.103Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e0f9f25f-180c-4ea3-8349-18b8d33e56c1", "backup": "backup-minio-3", "storage": "minio"} 2025-05-21T10:46:19.103Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e0f9f25f-180c-4ea3-8349-18b8d33e56c1", "backup": "backup-minio-3", "storage": "minio", "backupCmd": "backup [name: 2025-05-21T10:46:19Z, compression: gzip (level: default)] ", "profile": ""} 2025-05-21T10:46:19.109Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e0f9f25f-180c-4ea3-8349-18b8d33e56c1", "previous": "", "current": "requested"} 2025-05-21T10:46:19.138Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e0f9f25f-180c-4ea3-8349-18b8d33e56c1"} 2025-05-21T10:46:19.138Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "2327989e-adb9-442d-91df-3e562af24de1"} 2025-05-21T10:46:19.190Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "2327989e-adb9-442d-91df-3e562af24de1", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:19.190Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "2327989e-adb9-442d-91df-3e562af24de1", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:46:19.235Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "2327989e-adb9-442d-91df-3e562af24de1"} 2025-05-21T10:46:19.237Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "2327989e-adb9-442d-91df-3e562af24de1", "pbmName": "2025-05-21T10:46:18Z", "backup": "backup-minio-3"} 2025-05-21T10:46:19.238Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "2327989e-adb9-442d-91df-3e562af24de1"} 2025-05-21T10:46:19.238Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "aafedb01-b71d-464c-afed-a5b45cb5c07e"} 2025-05-21T10:46:19.284Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "aafedb01-b71d-464c-afed-a5b45cb5c07e", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:19.284Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "aafedb01-b71d-464c-afed-a5b45cb5c07e", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:46:19.333Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "aafedb01-b71d-464c-afed-a5b45cb5c07e"} 2025-05-21T10:46:19.334Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "aafedb01-b71d-464c-afed-a5b45cb5c07e", "pbmName": "2025-05-21T10:46:19Z", "backup": "backup-minio-3"} 2025-05-21T10:46:19.336Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "aafedb01-b71d-464c-afed-a5b45cb5c07e"} 2025-05-21T10:46:24.013Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "7155e713-392f-48ae-86ab-72704ce78c4e"} 2025-05-21T10:46:24.059Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "7155e713-392f-48ae-86ab-72704ce78c4e", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:24.059Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "7155e713-392f-48ae-86ab-72704ce78c4e", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:46:24.111Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "7155e713-392f-48ae-86ab-72704ce78c4e"} 2025-05-21T10:46:24.113Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "7155e713-392f-48ae-86ab-72704ce78c4e", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824379,"I":20},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.pitr-8397.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:46:24.117Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "7155e713-392f-48ae-86ab-72704ce78c4e"} 2025-05-21T10:46:29.118Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "3a432b8e-60bc-4405-aacb-8a6de13e6fa9"} 2025-05-21T10:46:29.162Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "3a432b8e-60bc-4405-aacb-8a6de13e6fa9", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:29.162Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "3a432b8e-60bc-4405-aacb-8a6de13e6fa9", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:46:29.212Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "3a432b8e-60bc-4405-aacb-8a6de13e6fa9"} 2025-05-21T10:46:29.213Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "3a432b8e-60bc-4405-aacb-8a6de13e6fa9", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:46:29.216Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "3a432b8e-60bc-4405-aacb-8a6de13e6fa9"} 2025-05-21T10:46:34.217Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e5daf7a6-e7e4-46e6-968b-2a15d4926fcc"} 2025-05-21T10:46:34.263Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e5daf7a6-e7e4-46e6-968b-2a15d4926fcc", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:34.263Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e5daf7a6-e7e4-46e6-968b-2a15d4926fcc", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:46:34.309Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e5daf7a6-e7e4-46e6-968b-2a15d4926fcc"} 2025-05-21T10:46:34.310Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e5daf7a6-e7e4-46e6-968b-2a15d4926fcc", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:46:34.314Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e5daf7a6-e7e4-46e6-968b-2a15d4926fcc"} 2025-05-21T10:46:39.314Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "ff67936c-0756-41be-a870-316c495b3018"} 2025-05-21T10:46:39.359Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "ff67936c-0756-41be-a870-316c495b3018", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:39.359Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "ff67936c-0756-41be-a870-316c495b3018", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:46:39.411Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "ff67936c-0756-41be-a870-316c495b3018"} 2025-05-21T10:46:39.413Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "ff67936c-0756-41be-a870-316c495b3018", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:46:39.416Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "ff67936c-0756-41be-a870-316c495b3018"} 2025-05-21T10:46:44.417Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14b686ff-8ed4-4c51-95e0-6697ea15dc07"} 2025-05-21T10:46:44.458Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14b686ff-8ed4-4c51-95e0-6697ea15dc07", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:44.458Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14b686ff-8ed4-4c51-95e0-6697ea15dc07", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:46:44.514Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14b686ff-8ed4-4c51-95e0-6697ea15dc07"} 2025-05-21T10:46:44.517Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14b686ff-8ed4-4c51-95e0-6697ea15dc07", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:46:44.520Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "14b686ff-8ed4-4c51-95e0-6697ea15dc07"} 2025-05-21T10:46:49.521Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "d0330a0c-4e16-4ef8-8304-2539eda26daa"} 2025-05-21T10:46:49.562Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "d0330a0c-4e16-4ef8-8304-2539eda26daa", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:49.562Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "d0330a0c-4e16-4ef8-8304-2539eda26daa", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:46:49.604Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "d0330a0c-4e16-4ef8-8304-2539eda26daa"} 2025-05-21T10:46:49.606Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "d0330a0c-4e16-4ef8-8304-2539eda26daa", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:46:49.608Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "d0330a0c-4e16-4ef8-8304-2539eda26daa"} 2025-05-21T10:46:54.609Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "c3a75921-8b5b-4371-8267-8325bfce6671"} 2025-05-21T10:46:54.656Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "c3a75921-8b5b-4371-8267-8325bfce6671", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:54.656Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "c3a75921-8b5b-4371-8267-8325bfce6671", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:46:54.703Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "c3a75921-8b5b-4371-8267-8325bfce6671"} 2025-05-21T10:46:54.705Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "c3a75921-8b5b-4371-8267-8325bfce6671", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:46:54.708Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "c3a75921-8b5b-4371-8267-8325bfce6671"} 2025-05-21T10:46:59.710Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "91a6f9ab-c63f-4463-8cb0-85c8f071a0db"} 2025-05-21T10:46:59.765Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "91a6f9ab-c63f-4463-8cb0-85c8f071a0db", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:46:59.765Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "91a6f9ab-c63f-4463-8cb0-85c8f071a0db", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:46:59.815Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "91a6f9ab-c63f-4463-8cb0-85c8f071a0db"} 2025-05-21T10:46:59.817Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "91a6f9ab-c63f-4463-8cb0-85c8f071a0db", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:46:59.820Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "91a6f9ab-c63f-4463-8cb0-85c8f071a0db"} 2025-05-21T10:47:04.821Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "c8b6d8a5-e1c1-45c5-a8c4-027a500f4600"} 2025-05-21T10:47:04.862Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "c8b6d8a5-e1c1-45c5-a8c4-027a500f4600", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:47:04.862Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "c8b6d8a5-e1c1-45c5-a8c4-027a500f4600", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:47:04.911Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "c8b6d8a5-e1c1-45c5-a8c4-027a500f4600"} 2025-05-21T10:47:04.913Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "c8b6d8a5-e1c1-45c5-a8c4-027a500f4600", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:47:04.916Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "c8b6d8a5-e1c1-45c5-a8c4-027a500f4600"} 2025-05-21T10:47:09.917Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "8b3fd8a4-12fe-4eaa-9289-6d681aeb74b5"} 2025-05-21T10:47:09.967Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "8b3fd8a4-12fe-4eaa-9289-6d681aeb74b5", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:47:09.967Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "8b3fd8a4-12fe-4eaa-9289-6d681aeb74b5", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:47:10.018Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "8b3fd8a4-12fe-4eaa-9289-6d681aeb74b5"} 2025-05-21T10:47:10.019Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "8b3fd8a4-12fe-4eaa-9289-6d681aeb74b5", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:47:10.023Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "8b3fd8a4-12fe-4eaa-9289-6d681aeb74b5"} 2025-05-21T10:47:15.024Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "833bbefc-c8b6-4bde-9eea-ae1545e45574"} 2025-05-21T10:47:15.072Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "833bbefc-c8b6-4bde-9eea-ae1545e45574", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:47:15.072Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "833bbefc-c8b6-4bde-9eea-ae1545e45574", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:47:15.122Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "833bbefc-c8b6-4bde-9eea-ae1545e45574"} 2025-05-21T10:47:15.124Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "833bbefc-c8b6-4bde-9eea-ae1545e45574", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:47:15.126Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "833bbefc-c8b6-4bde-9eea-ae1545e45574"} 2025-05-21T10:47:20.127Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "f0f6c9fe-09d7-4d41-b6fb-3cb0400899d7"} 2025-05-21T10:47:20.170Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "f0f6c9fe-09d7-4d41-b6fb-3cb0400899d7", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:47:20.170Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "f0f6c9fe-09d7-4d41-b6fb-3cb0400899d7", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:47:20.217Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "f0f6c9fe-09d7-4d41-b6fb-3cb0400899d7"} 2025-05-21T10:47:20.218Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "f0f6c9fe-09d7-4d41-b6fb-3cb0400899d7", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:47:20.221Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "f0f6c9fe-09d7-4d41-b6fb-3cb0400899d7"} 2025-05-21T10:47:25.223Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "71f81cd4-d849-4593-a5aa-7eeaf1f311d8"} 2025-05-21T10:47:25.282Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "71f81cd4-d849-4593-a5aa-7eeaf1f311d8", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:47:25.282Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "71f81cd4-d849-4593-a5aa-7eeaf1f311d8", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:47:25.328Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "71f81cd4-d849-4593-a5aa-7eeaf1f311d8"} 2025-05-21T10:47:25.330Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "71f81cd4-d849-4593-a5aa-7eeaf1f311d8", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:47:25.333Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "71f81cd4-d849-4593-a5aa-7eeaf1f311d8"} 2025-05-21T10:47:30.333Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "45446549-73d4-4a31-bc9e-547b53c93b5a"} 2025-05-21T10:47:30.378Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "45446549-73d4-4a31-bc9e-547b53c93b5a", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:47:30.378Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "45446549-73d4-4a31-bc9e-547b53c93b5a", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:47:30.421Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "45446549-73d4-4a31-bc9e-547b53c93b5a"} 2025-05-21T10:47:30.423Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "45446549-73d4-4a31-bc9e-547b53c93b5a", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:47:30.426Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "45446549-73d4-4a31-bc9e-547b53c93b5a"} 2025-05-21T10:47:35.427Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "5e94c695-ed22-4252-a7d1-21ce8fc3ab9b"} 2025-05-21T10:47:35.489Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "5e94c695-ed22-4252-a7d1-21ce8fc3ab9b", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:47:35.489Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "5e94c695-ed22-4252-a7d1-21ce8fc3ab9b", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:47:35.536Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "5e94c695-ed22-4252-a7d1-21ce8fc3ab9b"} 2025-05-21T10:47:35.538Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "5e94c695-ed22-4252-a7d1-21ce8fc3ab9b", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:47:35.541Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "5e94c695-ed22-4252-a7d1-21ce8fc3ab9b"} 2025-05-21T10:47:40.541Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "abc86c70-731e-4c4a-867f-b101bb1f9760"} 2025-05-21T10:47:40.586Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "abc86c70-731e-4c4a-867f-b101bb1f9760", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:47:40.586Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "abc86c70-731e-4c4a-867f-b101bb1f9760", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:47:40.638Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "abc86c70-731e-4c4a-867f-b101bb1f9760"} 2025-05-21T10:47:40.640Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "abc86c70-731e-4c4a-867f-b101bb1f9760", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:47:40.643Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "abc86c70-731e-4c4a-867f-b101bb1f9760"} 2025-05-21T10:47:45.644Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "6b707003-4161-4cb1-982c-5eb7783861ac"} 2025-05-21T10:47:45.692Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "6b707003-4161-4cb1-982c-5eb7783861ac", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:47:45.692Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "6b707003-4161-4cb1-982c-5eb7783861ac", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:47:45.740Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "6b707003-4161-4cb1-982c-5eb7783861ac"} 2025-05-21T10:47:45.742Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "6b707003-4161-4cb1-982c-5eb7783861ac", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:47:45.745Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "6b707003-4161-4cb1-982c-5eb7783861ac"} 2025-05-21T10:47:50.745Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "310e9b36-260e-4542-bbfc-bb979eec0adc"} 2025-05-21T10:47:50.792Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "310e9b36-260e-4542-bbfc-bb979eec0adc", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:47:50.792Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "310e9b36-260e-4542-bbfc-bb979eec0adc", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:47:50.840Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "310e9b36-260e-4542-bbfc-bb979eec0adc"} 2025-05-21T10:47:50.842Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "310e9b36-260e-4542-bbfc-bb979eec0adc", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:47:50.844Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "310e9b36-260e-4542-bbfc-bb979eec0adc"} 2025-05-21T10:47:55.845Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "64041681-69ae-48c5-9d2c-e7eb8a96dd88"} 2025-05-21T10:47:55.891Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "64041681-69ae-48c5-9d2c-e7eb8a96dd88", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:47:55.891Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "64041681-69ae-48c5-9d2c-e7eb8a96dd88", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:47:55.933Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "64041681-69ae-48c5-9d2c-e7eb8a96dd88"} 2025-05-21T10:47:55.934Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "64041681-69ae-48c5-9d2c-e7eb8a96dd88", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:47:55.937Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "64041681-69ae-48c5-9d2c-e7eb8a96dd88"} 2025-05-21T10:48:00.938Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "1730d660-4c30-4df4-810e-464685f44c41"} 2025-05-21T10:48:00.987Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "1730d660-4c30-4df4-810e-464685f44c41", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:48:00.987Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "1730d660-4c30-4df4-810e-464685f44c41", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:48:01.057Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "1730d660-4c30-4df4-810e-464685f44c41"} 2025-05-21T10:48:01.059Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "1730d660-4c30-4df4-810e-464685f44c41", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:48:01.062Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "1730d660-4c30-4df4-810e-464685f44c41"} 2025-05-21T10:48:06.062Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "79aaec95-93ad-4822-857a-ab41e7444bbc"} 2025-05-21T10:48:06.105Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "79aaec95-93ad-4822-857a-ab41e7444bbc", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:48:06.105Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "79aaec95-93ad-4822-857a-ab41e7444bbc", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:48:06.150Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "79aaec95-93ad-4822-857a-ab41e7444bbc"} 2025-05-21T10:48:06.152Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "79aaec95-93ad-4822-857a-ab41e7444bbc", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:48:06.155Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "79aaec95-93ad-4822-857a-ab41e7444bbc"} 2025-05-21T10:48:11.156Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e43da1b8-bfbc-4d88-8ca6-e6f5c658edd4"} 2025-05-21T10:48:11.207Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e43da1b8-bfbc-4d88-8ca6-e6f5c658edd4", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:48:11.207Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e43da1b8-bfbc-4d88-8ca6-e6f5c658edd4", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:48:11.252Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e43da1b8-bfbc-4d88-8ca6-e6f5c658edd4"} 2025-05-21T10:48:11.254Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e43da1b8-bfbc-4d88-8ca6-e6f5c658edd4", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:48:11.257Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "e43da1b8-bfbc-4d88-8ca6-e6f5c658edd4"} 2025-05-21T10:48:16.259Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "4a0382d9-2bb7-41cf-8b55-8aeb21cbca21"} 2025-05-21T10:48:16.305Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "4a0382d9-2bb7-41cf-8b55-8aeb21cbca21", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:48:16.305Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "4a0382d9-2bb7-41cf-8b55-8aeb21cbca21", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:48:16.354Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "4a0382d9-2bb7-41cf-8b55-8aeb21cbca21"} 2025-05-21T10:48:16.356Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "4a0382d9-2bb7-41cf-8b55-8aeb21cbca21", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:48:16.359Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "4a0382d9-2bb7-41cf-8b55-8aeb21cbca21"} 2025-05-21T10:48:21.360Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "84bb0feb-4ea5-4ec0-b56a-d5c0d1639b32"} 2025-05-21T10:48:21.402Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "84bb0feb-4ea5-4ec0-b56a-d5c0d1639b32", "cluster": "some-name", "namespace": "pitr-8397"} 2025-05-21T10:48:21.402Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "84bb0feb-4ea5-4ec0-b56a-d5c0d1639b32", "currentJob": {"Name":"backup-minio-3","Type":0}} 2025-05-21T10:48:21.451Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "84bb0feb-4ea5-4ec0-b56a-d5c0d1639b32"} 2025-05-21T10:48:21.453Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "84bb0feb-4ea5-4ec0-b56a-d5c0d1639b32", "backup": "backup-minio-3", "pbmName": "2025-05-21T10:46:19Z", "meta": {"type":"logical","opid":"682daefb7bb1c7ef38a9e566","name":"2025-05-21T10:46:19Z","replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824379,"last_transition_ts":1747824379,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824384,"I":10},"status":"starting","conditions":[{"timestamp":1747824379,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-8397.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"off"}} 2025-05-21T10:48:21.456Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "84bb0feb-4ea5-4ec0-b56a-d5c0d1639b32", "previous": "requested", "current": "error"} 2025-05-21T10:48:21.479Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "84bb0feb-4ea5-4ec0-b56a-d5c0d1639b32", "lease": "psmdb-some-name-backup-lock"} 2025-05-21T10:48:21.502Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "84bb0feb-4ea5-4ec0-b56a-d5c0d1639b32"} 2025-05-21T10:48:21.502Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "6a2f41e2-7243-4d2f-9c09-568928a35119"} 2025-05-21T10:48:21.503Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-3","namespace":"pitr-8397"}, "namespace": "pitr-8397", "name": "backup-minio-3", "reconcileID": "6a2f41e2-7243-4d2f-9c09-568928a35119"} NAME CLUSTER STORAGE DESTINATION TYPE STATUS COMPLETED AGE backup-minio-0 some-name minio s3://operator-testing/2025-05-21T10:33:27Z logical ready 14m 15m backup-minio-1 some-name minio s3://operator-testing/2025-05-21T10:37:33Z logical ready 10m 10m backup-minio-2 some-name minio s3://operator-testing/2025-05-21T10:42:03Z logical ready 6m17s 6m23s backup-minio-3 some-name minio s3://operator-testing/2025-05-21T10:46:19Z logical error 2m8s Backup object psmdb-backup/backup-minio-3 is in error state. something went wrong with operator or kubernetes cluster