Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/logs/demand-backup-incremental-sharded.log
grep: warning: stray \ before -
Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1
+ create_infra demand-backup-incremental-sharded-2770
+ local ns=demand-backup-incremental-sharded-2770
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.5iNLiCsMhr
++ mktemp
+ local LAST_ERR=/tmp/tmp.G2Y0e0dMeG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5iNLiCsMhr
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.G2Y0e0dMeG
+ rm /tmp/tmp.5iNLiCsMhr /tmp/tmp.G2Y0e0dMeG
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-17530 backup-aws-s3-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-17530 backup-azure-blob-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-17530 backup-gcp-cs-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-17530 backup-minio-not-base-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-not-base-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-17530 backup-minio-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.eZCP4Wx9fY
++ mktemp
+ local LAST_ERR=/tmp/tmp.YdkvO7KJim
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eZCP4Wx9fY
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.YdkvO7KJim
+ rm /tmp/tmp.eZCP4Wx9fY /tmp/tmp.YdkvO7KJim
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.YNiWx8hqw7
++ mktemp
+ local LAST_ERR=/tmp/tmp.uGSZi04g1i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YNiWx8hqw7
+ cat /tmp/tmp.uGSZi04g1i
+ rm /tmp/tmp.YNiWx8hqw7 /tmp/tmp.uGSZi04g1i
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Np5h6poPWj
++ mktemp
+ local LAST_ERR=/tmp/tmp.LvqxciYraM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Np5h6poPWj
+ cat /tmp/tmp.LvqxciYraM
+ rm /tmp/tmp.Np5h6poPWj /tmp/tmp.LvqxciYraM
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.ks85egubva
++ mktemp
+ local LAST_ERR=/tmp/tmp.k442azqKTN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ks85egubva
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.k442azqKTN
+ rm /tmp/tmp.ks85egubva /tmp/tmp.k442azqKTN
+ return 0
+ check_crd_for_deletion PR-1987-d120637a
+ local git_tag=PR-1987-d120637a
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1987-d120637a/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6Z3W15ydHO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ywR7YfIaYO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.6Z3W15ydHO
++ cat /tmp/tmp.ywR7YfIaYO
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.6Z3W15ydHO
++ cat /tmp/tmp.ywR7YfIaYO
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.6Z3W15ydHO
++ cat /tmp/tmp.ywR7YfIaYO
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.6Z3W15ydHO
++ cat /tmp/tmp.ywR7YfIaYO
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.6Z3W15ydHO /tmp/tmp.ywR7YfIaYO
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ xargs kubectl delete ns
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_OUT=/tmp/tmp.aJE1076rec
++ mktemp
+ local LAST_OUT=/tmp/tmp.dW067mK6tJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.0O5LkW6Xcs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.tbwjZukTTr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aJE1076rec
+ cat /tmp/tmp.0O5LkW6Xcs
+ rm /tmp/tmp.aJE1076rec /tmp/tmp.0O5LkW6Xcs
+ return 0
namespace "demand-backup-incremental-sharded-17530" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dW067mK6tJ
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.tbwjZukTTr
+ rm /tmp/tmp.dW067mK6tJ /tmp/tmp.tbwjZukTTr
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.PWAJ2Tj06H
++ mktemp
+ local LAST_ERR=/tmp/tmp.kgxbNLWlQD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PWAJ2Tj06H
+ cat /tmp/tmp.kgxbNLWlQD
+ rm /tmp/tmp.PWAJ2Tj06H /tmp/tmp.kgxbNLWlQD
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.CSk8TGxOll
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZFQoQWi4oY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CSk8TGxOll
namespace/psmdb-operator created
+ cat /tmp/tmp.ZFQoQWi4oY
+ rm /tmp/tmp.CSk8TGxOll /tmp/tmp.ZFQoQWi4oY
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wIrC7FGYu9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7Puc0oCZHg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wIrC7FGYu9
++ cat /tmp/tmp.7Puc0oCZHg
++ rm /tmp/tmp.wIrC7FGYu9 /tmp/tmp.7Puc0oCZHg
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1987-d120637a-5-cluster7 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.EmmeAZIbpd
++ mktemp
+ local LAST_ERR=/tmp/tmp.d2jFCVFwB8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1987-d120637a-5-cluster7 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EmmeAZIbpd
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1987-d120637a-5-cluster7" modified.
+ cat /tmp/tmp.d2jFCVFwB8
+ rm /tmp/tmp.EmmeAZIbpd /tmp/tmp.d2jFCVFwB8
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.79r55ngmDe
++ mktemp
+ local LAST_ERR=/tmp/tmp.2WV9VlWD2r
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.79r55ngmDe
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.2WV9VlWD2r
+ rm /tmp/tmp.79r55ngmDe /tmp/tmp.2WV9VlWD2r
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.VoJRZfh7Ky
++ mktemp
+ local LAST_ERR=/tmp/tmp.1MTXWLTIVx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VoJRZfh7Ky
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.1MTXWLTIVx
+ rm /tmp/tmp.VoJRZfh7Ky /tmp/tmp.1MTXWLTIVx
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1987-d120637a") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.YLbzKI7hKP
++ mktemp
+ local LAST_ERR=/tmp/tmp.WYDjeEOJx9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YLbzKI7hKP
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.WYDjeEOJx9
+ rm /tmp/tmp.YLbzKI7hKP /tmp/tmp.WYDjeEOJx9
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LFv1LKjZC9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AcX8S2KYVp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LFv1LKjZC9
++ cat /tmp/tmp.AcX8S2KYVp
++ rm /tmp/tmp.LFv1LKjZC9 /tmp/tmp.AcX8S2KYVp
++ return 0
+ wait_pod percona-server-mongodb-operator-85f675667c-5lgsm
+ local pod=percona-server-mongodb-operator-85f675667c-5lgsm
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-85f675667c-5lgsm to be ready..OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bvkY94FQ4k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9MpsmxLmDS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bvkY94FQ4k
++ cat /tmp/tmp.9MpsmxLmDS
++ rm /tmp/tmp.bvkY94FQ4k /tmp/tmp.9MpsmxLmDS
++ return 0
+ kubectl_bin logs percona-server-mongodb-operator-85f675667c-5lgsm
++ mktemp
+ local LAST_OUT=/tmp/tmp.i8znqppu4x
++ mktemp
+ local LAST_ERR=/tmp/tmp.B7yfUF1toe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs percona-server-mongodb-operator-85f675667c-5lgsm
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.i8znqppu4x
+ cat /tmp/tmp.B7yfUF1toe
+ rm /tmp/tmp.i8znqppu4x /tmp/tmp.B7yfUF1toe
+ return 0
2025-09-22T14:34:57.115Z	INFO	setup	Manager starting up	{"gitCommit": "d120637a46cfaf89d946ade2d9faedf115796e14", "gitBranch": "PR-1987-d120637a", "buildTime": "", "goVersion": "go1.25.1", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-incremental-sharded-2770
+ local namespace=demand-backup-incremental-sharded-2770
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces demand-backup-incremental-sharded-2770'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-incremental-sharded-2770
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-incremental-sharded-2770 --ignore-not-found
++ mktemp
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.FG4mfQe7CH
++ mktemp
+ local LAST_OUT=/tmp/tmp.vnCrTUGuDu
+ local LAST_ERR=/tmp/tmp.gyEQQM6ltj
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.6F60m9HoJM
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-incremental-sharded-2770 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FG4mfQe7CH
+ cat /tmp/tmp.gyEQQM6ltj
+ rm /tmp/tmp.FG4mfQe7CH /tmp/tmp.gyEQQM6ltj
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vnCrTUGuDu
+ cat /tmp/tmp.6F60m9HoJM
+ rm /tmp/tmp.vnCrTUGuDu /tmp/tmp.6F60m9HoJM
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-incremental-sharded-2770
++ mktemp
+ local LAST_OUT=/tmp/tmp.7EyAUB9lFh
++ mktemp
+ local LAST_ERR=/tmp/tmp.MOQoYrIeK9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-incremental-sharded-2770
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7EyAUB9lFh
+ cat /tmp/tmp.MOQoYrIeK9
+ rm /tmp/tmp.7EyAUB9lFh /tmp/tmp.MOQoYrIeK9
+ return 0
+ desc 'create namespace demand-backup-incremental-sharded-2770'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-incremental-sharded-2770
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-incremental-sharded-2770
++ mktemp
+ local LAST_OUT=/tmp/tmp.AxSorOLaKR
++ mktemp
+ local LAST_ERR=/tmp/tmp.WDUHdUxHeK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-incremental-sharded-2770
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AxSorOLaKR
namespace/demand-backup-incremental-sharded-2770 created
+ cat /tmp/tmp.WDUHdUxHeK
+ rm /tmp/tmp.AxSorOLaKR /tmp/tmp.WDUHdUxHeK
+ return 0
+ set_kube_ctx demand-backup-incremental-sharded-2770
+ local namespace=demand-backup-incremental-sharded-2770
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gGaYQiOrtI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sY2MnnM3op
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gGaYQiOrtI
++ cat /tmp/tmp.sY2MnnM3op
++ rm /tmp/tmp.gGaYQiOrtI /tmp/tmp.sY2MnnM3op
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1987-d120637a-5-cluster7 --namespace=demand-backup-incremental-sharded-2770
++ mktemp
+ local LAST_OUT=/tmp/tmp.0Tn0JsnoCM
++ mktemp
+ local LAST_ERR=/tmp/tmp.sIOEba8kQo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1987-d120637a-5-cluster7 --namespace=demand-backup-incremental-sharded-2770
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0Tn0JsnoCM
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1987-d120637a-5-cluster7" modified.
+ cat /tmp/tmp.sIOEba8kQo
+ rm /tmp/tmp.0Tn0JsnoCM /tmp/tmp.sIOEba8kQo
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Mon Sep 22 14:35:16 2025
NAMESPACE: demand-backup-incremental-sharded-2770
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-incremental-sharded-2770.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-incremental-sharded-2770 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-incremental-sharded-2770

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-incremental-sharded-2770 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-incremental-sharded-2770 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3CVmdoCIqf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.I6Ddh0s8fK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3CVmdoCIqf
++ cat /tmp/tmp.I6Ddh0s8fK
++ rm /tmp/tmp.3CVmdoCIqf /tmp/tmp.I6Ddh0s8fK
++ return 0
+ MINIO_POD=minio-service-86dfccd949-cqjkp
+ wait_pod minio-service-86dfccd949-cqjkp
+ local pod=minio-service-86dfccd949-cqjkp
+ set +o xtrace
waiting for pod/minio-service-86dfccd949-cqjkp to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-incremental-sharded-2770.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.CCqGHg7MIa
++ mktemp
+ local LAST_ERR=/tmp/tmp.LjIkVdZXc5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-incremental-sharded-2770.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CCqGHg7MIa
service/minio-service created
+ cat /tmp/tmp.LjIkVdZXc5
+ rm /tmp/tmp.CCqGHg7MIa /tmp/tmp.LjIkVdZXc5
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Uzh6vQYgiJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.cGhG4FKw8y
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Uzh6vQYgiJ
pod "aws-cli" deleted from demand-backup-incremental-sharded-2770 namespace
+ cat /tmp/tmp.cGhG4FKw8y
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.Uzh6vQYgiJ /tmp/tmp.cGhG4FKw8y
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.KLnthT709Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.f36IpJbjC6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KLnthT709Q
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.f36IpJbjC6
+ rm /tmp/tmp.KLnthT709Q /tmp/tmp.f36IpJbjC6
+ return 0
+ desc 'Testing on sharded cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Testing on sharded cluster
-----------------------------------------------------------------------------------

+ log 'Creating PSMDB cluster'
+ set +o xtrace
[2025-09-22T14:36:02+0000] Creating PSMDB cluster
+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ddxf73h4At
++ mktemp
+ local LAST_ERR=/tmp/tmp.HD7KmAMN3U
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ddxf73h4At
secret/some-users created
+ cat /tmp/tmp.HD7KmAMN3U
+ rm /tmp/tmp.ddxf73h4At /tmp/tmp.HD7KmAMN3U
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ local LAST_OUT=/tmp/tmp.LPwCrAvjTi
++ mktemp
+ local LAST_ERR=/tmp/tmp.u3UzD9E1v9
+ local exit_status=0
+ local timeout=4
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1987-d120637a"'
++ seq 0 2
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LPwCrAvjTi
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.u3UzD9E1v9
+ rm /tmp/tmp.LPwCrAvjTi /tmp/tmp.u3UzD9E1v9
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.QkDR4bo2BU
++ mktemp
+ local LAST_ERR=/tmp/tmp.4tmTaJurmW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QkDR4bo2BU
deployment.apps/psmdb-client created
+ cat /tmp/tmp.4tmTaJurmW
+ rm /tmp/tmp.QkDR4bo2BU /tmp/tmp.4tmTaJurmW
+ return 0
+ log 'check if all pods started'
+ set +o xtrace
[2025-09-22T14:36:08+0000] check if all pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.misSRCnlt5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wIZk0csMIo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.misSRCnlt5
++ cat /tmp/tmp.wIZk0csMIo
++ rm /tmp/tmp.misSRCnlt5 /tmp/tmp.wIZk0csMIo
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ygPrtR7KRm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PttDi6VCFE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ygPrtR7KRm
++ cat /tmp/tmp.PttDi6VCFE
++ rm /tmp/tmp.ygPrtR7KRm /tmp/tmp.PttDi6VCFE
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GzwC2ZjVk4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uQsEOIIxSI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GzwC2ZjVk4
++ cat /tmp/tmp.uQsEOIIxSI
++ rm /tmp/tmp.GzwC2ZjVk4 /tmp/tmp.uQsEOIIxSI
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness......................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TAGTyMT8kn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vP3nwMAFy5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TAGTyMT8kn
++ cat /tmp/tmp.vP3nwMAFy5
++ rm /tmp/tmp.TAGTyMT8kn /tmp/tmp.vP3nwMAFy5
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D74fp8bIsd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pVFlyeDnd7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D74fp8bIsd
++ cat /tmp/tmp.pVFlyeDnd7
++ rm /tmp/tmp.D74fp8bIsd /tmp/tmp.pVFlyeDnd7
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QP09A2Cr8Z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BS7MPUYORq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QP09A2Cr8Z
++ cat /tmp/tmp.BS7MPUYORq
++ rm /tmp/tmp.QP09A2Cr8Z /tmp/tmp.BS7MPUYORq
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RSj6OsqQU0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.C12ov1Pzuh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RSj6OsqQU0
++ cat /tmp/tmp.C12ov1Pzuh
++ rm /tmp/tmp.RSj6OsqQU0 /tmp/tmp.C12ov1Pzuh
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u8CnNg4TRy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AVEE0I1LIu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.u8CnNg4TRy
++ cat /tmp/tmp.AVEE0I1LIu
++ rm /tmp/tmp.u8CnNg4TRy /tmp/tmp.AVEE0I1LIu
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FbxIHgAI58
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kYmPQ8iDLu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FbxIHgAI58
++ cat /tmp/tmp.kYmPQ8iDLu
++ rm /tmp/tmp.FbxIHgAI58 /tmp/tmp.kYmPQ8iDLu
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HsJodbrtyT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tj07Plgvxd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HsJodbrtyT
++ cat /tmp/tmp.tj07Plgvxd
++ rm /tmp/tmp.HsJodbrtyT /tmp/tmp.tj07Plgvxd
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

++ kubectl_bin get svc some-name-mongos '-o=jsonpath={.status}'
++ jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0] | if .ip then .ip else .hostname end'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OMerUT6jl5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qTfk2wyhP5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get svc some-name-mongos '-o=jsonpath={.status}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OMerUT6jl5
++ cat /tmp/tmp.qTfk2wyhP5
++ rm /tmp/tmp.OMerUT6jl5 /tmp/tmp.qTfk2wyhP5
++ return 0
+ lbEndpoint=35.239.64.210
+ '[' -z 35.239.64.210 ']'
+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-2770
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-2770
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IDJMlz7ePc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qyBPKds7Wv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IDJMlz7ePc
++ cat /tmp/tmp.qyBPKds7Wv
++ rm /tmp/tmp.IDJMlz7ePc /tmp/tmp.qyBPKds7Wv
++ return 0
+ local client_container=psmdb-client-b9788d8bc-h62zq
+ kubectl_bin exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.td24FcAFoq
++ mktemp
+ local LAST_ERR=/tmp/tmp.kKASKsYgBH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.td24FcAFoq
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("3f11cb9a-f808-4947-bef3-df655c1e971f") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.kKASKsYgBH
+ rm /tmp/tmp.td24FcAFoq /tmp/tmp.kKASKsYgBH
+ return 0
+ sleep 1
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.psbsXIqQah
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aJXZWbW5nG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.psbsXIqQah
++ cat /tmp/tmp.aJXZWbW5nG
++ rm /tmp/tmp.psbsXIqQah /tmp/tmp.aJXZWbW5nG
++ return 0
+ local client_container=psmdb-client-b9788d8bc-h62zq
+ kubectl_bin exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.BCNkWzyZzq
++ mktemp
+ local LAST_ERR=/tmp/tmp.uu4qTpmbbH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BCNkWzyZzq
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("bf49aff8-162d-4521-ae0f-9faa6d2b28b9") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.uu4qTpmbbH
+ rm /tmp/tmp.BCNkWzyZzq /tmp/tmp.uu4qTpmbbH
+ return 0
+ sleep 5
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770 mongodb '' '' 27017
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wVisUAcogt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fp9bI58KnB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wVisUAcogt
++ cat /tmp/tmp.fp9bI58KnB
++ rm /tmp/tmp.wVisUAcogt /tmp/tmp.fp9bI58KnB
++ return 0
+ local client_container=psmdb-client-b9788d8bc-h62zq
+ kubectl_bin exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.shigfR2Ley
++ mktemp
+ local LAST_ERR=/tmp/tmp.Q1aM7z4Kov
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.shigfR2Ley
+ cat /tmp/tmp.Q1aM7z4Kov
+ rm /tmp/tmp.shigfR2Ley /tmp/tmp.Q1aM7z4Kov
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.Ui1CqFHLwe/find-sharded
+ log 'waiting 60 seconds for stable timestamp in wiredtiger'
+ set +o xtrace
[2025-09-22T14:39:24+0000] waiting 60 seconds for stable timestamp in wiredtiger
+ sleep 80
+ log 'running backups'
+ set +o xtrace
[2025-09-22T14:40:44+0000] running backups
+ '[' -z '' ']'
+ backup_name_aws=backup-aws-s3-sharded
+ backup_name_gcp=backup-gcp-cs-sharded
+ backup_name_azure=backup-azure-blob-sharded
+ run_backup aws-s3 backup-aws-s3-sharded
+ local storage=aws-s3
+ local backup_name=backup-aws-s3-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: backup-aws-s3-sharded/'
+ /usr/sbin/sed -e 's/storageName:/storageName: aws-s3/'
+ yq '.spec.type="incremental-base"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.46AXQ7g7DS
++ mktemp
+ local LAST_ERR=/tmp/tmp.UB9bolsrGo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.46AXQ7g7DS
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded created
+ cat /tmp/tmp.UB9bolsrGo
+ rm /tmp/tmp.46AXQ7g7DS /tmp/tmp.UB9bolsrGo
+ return 0
+ run_backup gcp-cs backup-gcp-cs-sharded
+ local storage=gcp-cs
+ local backup_name=backup-gcp-cs-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: backup-gcp-cs-sharded/'
+ /usr/sbin/sed -e 's/storageName:/storageName: gcp-cs/'
+ yq '.spec.type="incremental-base"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.6zXT8c3hLi
++ mktemp
+ local LAST_ERR=/tmp/tmp.xLjLl1JWNv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6zXT8c3hLi
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded created
+ cat /tmp/tmp.xLjLl1JWNv
+ rm /tmp/tmp.6zXT8c3hLi /tmp/tmp.xLjLl1JWNv
+ return 0
+ run_backup azure-blob backup-azure-blob-sharded
+ local storage=azure-blob
+ local backup_name=backup-azure-blob-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: backup-azure-blob-sharded/'
+ /usr/sbin/sed -e 's/storageName:/storageName: azure-blob/'
+ yq '.spec.type="incremental-base"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.GVevWhmiix
++ mktemp
+ local LAST_ERR=/tmp/tmp.sBxftaUdQj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GVevWhmiix
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded created
+ cat /tmp/tmp.sBxftaUdQj
+ rm /tmp/tmp.GVevWhmiix /tmp/tmp.sBxftaUdQj
+ return 0
+ wait_backup backup-aws-s3-sharded
+ local backup_name=backup-aws-s3-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3-sharded to reach ready state...........
+ check_backup_in_storage backup-aws-s3-sharded s3 rs0
+ local backup=backup-aws-s3-sharded
+ local storage_type=s3
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=s3.amazonaws.com
++ get_backup_dest backup-aws-s3-sharded
++ local backup_name=backup-aws-s3-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OSoXeub0Lw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vzwvz382VQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OSoXeub0Lw
++ cat /tmp/tmp.Vzwvz382VQ
++ rm /tmp/tmp.OSoXeub0Lw /tmp/tmp.Vzwvz382VQ
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:40:47Z
+ [[ s3 == \m\i\n\i\o ]]
+ local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:40:47Z/rs0/filelist.pbm
+ log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:40:47Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-09-22T14:41:15+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:40:47Z/rs0/filelist.pbm exists
+ curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:40:47Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 10067    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
x-amz-id-2: afWyPxciFM/70IerQY5vgWWKvVfJm8v9/kcoTB/KngpNCtMYAi2nf1Eh2+0MU/ZYWNplmhDfTJI=
x-amz-request-id: 53K32CH0P9G1S7TC
Date: Mon, 22 Sep 2025 14:41:17 GMT
Last-Modified: Mon, 22 Sep 2025 14:41:04 GMT
x-amz-expiration: expiry-date="Wed, 24 Sep 2025 00:00:00 GMT", rule-id="1 Days Cleanup"
ETag: "3b6080f0769c2fc692c55367991fef7f"
x-amz-server-side-encryption: AES256
Accept-Ranges: bytes
Content-Type: application/octet-stream
Content-Length: 10067
Server: AmazonS3

+ check_backup_in_storage backup-aws-s3-sharded s3 cfg
+ local backup=backup-aws-s3-sharded
+ local storage_type=s3
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=s3.amazonaws.com
++ get_backup_dest backup-aws-s3-sharded
++ local backup_name=backup-aws-s3-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GZGfaZ8Zua
++ sed 's|azure://||'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Toxge5WN0y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GZGfaZ8Zua
++ cat /tmp/tmp.Toxge5WN0y
++ rm /tmp/tmp.GZGfaZ8Zua /tmp/tmp.Toxge5WN0y
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:40:47Z
+ [[ s3 == \m\i\n\i\o ]]
+ local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:40:47Z/cfg/filelist.pbm
+ log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:40:47Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-09-22T14:41:17+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:40:47Z/cfg/filelist.pbm exists
+ curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:40:47Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 18299    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
x-amz-id-2: qQP/s0P2oqUer60IDbQDLgtGHpxET9m5zIzEzBUoMNJ0D0zHfczf1YliSEAXEkZrdeRA56/h7sQ=
x-amz-request-id: QYS9FXT2GMT1CXGS
Date: Mon, 22 Sep 2025 14:41:18 GMT
Last-Modified: Mon, 22 Sep 2025 14:41:10 GMT
x-amz-expiration: expiry-date="Wed, 24 Sep 2025 00:00:00 GMT", rule-id="1 Days Cleanup"
ETag: "11f27a3d172f16bf887e38615fc55ef7"
x-amz-server-side-encryption: AES256
Accept-Ranges: bytes
Content-Type: application/octet-stream
Content-Length: 18299
Server: AmazonS3

+ wait_backup backup-gcp-cs-sharded
+ local backup_name=backup-gcp-cs-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sharded to reach ready state............
+ check_backup_in_storage backup-gcp-cs-sharded gcs rs0
+ local backup=backup-gcp-cs-sharded
+ local storage_type=gcs
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sharded
++ local backup_name=backup-gcp-cs-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|s3://||'
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.dpI0YQSPAd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k6zVmcauqZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dpI0YQSPAd
++ cat /tmp/tmp.k6zVmcauqZ
++ rm /tmp/tmp.dpI0YQSPAd /tmp/tmp.k6zVmcauqZ
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:14Z
+ [[ gcs == \m\i\n\i\o ]]
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:14Z/rs0/filelist.pbm
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:14Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-09-22T14:41:43+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:14Z/rs0/filelist.pbm exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:14Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 10867    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 10867    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: AAwnv3IfMTmysHI9lqv0JT98Vez3bYfzb3drWZsvKo1c7Na7N-PR43uhVXAfZyVQYNB_zqb-
expires: Mon, 22 Sep 2025 15:41:43 GMT
date: Mon, 22 Sep 2025 14:41:43 GMT
cache-control: public, max-age=3600
last-modified: Mon, 22 Sep 2025 14:41:32 GMT
etag: "19e9a53a4d045a5ffcf2e957b8e63696"
x-goog-generation: 1758552092801973
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 10867
x-goog-hash: crc32c=9SRGnQ==
x-goog-hash: md5=GemlOk0EWl/88ulXuOY2lg==
x-goog-expiration: Tue, 23 Sep 2025 14:41:32 GMT
x-goog-storage-class: REGIONAL
accept-ranges: bytes
content-length: 10867
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ check_backup_in_storage backup-gcp-cs-sharded gcs cfg
+ local backup=backup-gcp-cs-sharded
+ local storage_type=gcs
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sharded
++ local backup_name=backup-gcp-cs-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.xH48Huj4vR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HSogwyhy4C
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xH48Huj4vR
++ cat /tmp/tmp.HSogwyhy4C
++ rm /tmp/tmp.xH48Huj4vR /tmp/tmp.HSogwyhy4C
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:14Z
+ [[ gcs == \m\i\n\i\o ]]
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:14Z/cfg/filelist.pbm
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:14Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-09-22T14:41:44+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:14Z/cfg/filelist.pbm exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:14Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 19099    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: AAwnv3Jb8AnyGU2oZdGew4FVjbUFjuNBI7jsQk1-af8AKgOI0PiInypQOBM4DZh9BgaWraJZR-D2i9M
expires: Mon, 22 Sep 2025 15:41:44 GMT
date: Mon, 22 Sep 2025 14:41:44 GMT
cache-control: public, max-age=3600
last-modified: Mon, 22 Sep 2025 14:41:37 GMT
etag: "07e18ac31f54fb64199ed175a60b2a41"
x-goog-generation: 1758552097775537
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 19099
x-goog-hash: crc32c=I7M5ng==
x-goog-hash: md5=B+GKwx9U+2QZntF1pgsqQQ==
x-goog-expiration: Tue, 23 Sep 2025 14:41:37 GMT
x-goog-storage-class: REGIONAL
accept-ranges: bytes
content-length: 19099
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ wait_backup backup-azure-blob-sharded
+ local backup_name=backup-azure-blob-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob-sharded to reach ready state.............
+ check_backup_in_storage backup-azure-blob-sharded azure rs0
+ local backup=backup-azure-blob-sharded
+ local storage_type=azure
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=engk8soperators.blob.core.windows.net
++ get_backup_dest backup-azure-blob-sharded
++ local backup_name=backup-azure-blob-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ipTNuEQU1P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KZ1KJAgVhg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ipTNuEQU1P
++ cat /tmp/tmp.KZ1KJAgVhg
++ rm /tmp/tmp.ipTNuEQU1P /tmp/tmp.KZ1KJAgVhg
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:43Z
+ [[ azure == \m\i\n\i\o ]]
+ local url=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:43Z/rs0/filelist.pbm
+ log 'checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:43Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-09-22T14:42:12+0000] checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:43Z/rs0/filelist.pbm exists
+ curl --fail --head https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:43Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 13697    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
Content-Length: 13697
Content-Type: application/octet-stream
Content-MD5: Or0suCNbNk7Xp5QEodo0tg==
Last-Modified: Mon, 22 Sep 2025 14:42:04 GMT
ETag: 0x8DDF9E632B89CC9
Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
x-ms-request-id: a434b89e-201e-0053-11cf-2bfc5b000000
x-ms-version: 2009-09-19
x-ms-lease-status: unlocked
x-ms-blob-type: BlockBlob
Date: Mon, 22 Sep 2025 14:42:12 GMT

+ check_backup_in_storage backup-azure-blob-sharded azure cfg
+ local backup=backup-azure-blob-sharded
+ local storage_type=azure
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=engk8soperators.blob.core.windows.net
++ get_backup_dest backup-azure-blob-sharded
++ local backup_name=backup-azure-blob-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.agx7VMNhsq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HirK6Y0WUM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.agx7VMNhsq
++ cat /tmp/tmp.HirK6Y0WUM
++ rm /tmp/tmp.agx7VMNhsq /tmp/tmp.HirK6Y0WUM
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:43Z
+ [[ azure == \m\i\n\i\o ]]
+ local url=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:43Z/cfg/filelist.pbm
+ log 'checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:43Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-09-22T14:42:14+0000] checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:43Z/cfg/filelist.pbm exists
+ curl --fail --head https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-09-22T14:41:43Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 18299    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
Content-Length: 18299
Content-Type: application/octet-stream
Content-MD5: zCiYHuVUqBGSbfc+3CLwIw==
Last-Modified: Mon, 22 Sep 2025 14:41:59 GMT
ETag: 0x8DDF9E62FF6D7E7
Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
x-ms-request-id: b96791b3-b01e-000c-30cf-2b4867000000
x-ms-version: 2009-09-19
x-ms-lease-status: unlocked
x-ms-blob-type: BlockBlob
Date: Mon, 22 Sep 2025 14:42:13 GMT

+ backup_name_minio=backup-minio-sharded
+ run_backup minio backup-minio-sharded
+ local storage=minio
+ local backup_name=backup-minio-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-sharded/'
+ /usr/sbin/sed -e 's/storageName:/storageName: minio/'
+ yq '.spec.type="incremental-base"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.qMeTb7XVk5
++ mktemp
+ local LAST_ERR=/tmp/tmp.V6w8TgTsl3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qMeTb7XVk5
perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded created
+ cat /tmp/tmp.V6w8TgTsl3
+ rm /tmp/tmp.qMeTb7XVk5 /tmp/tmp.V6w8TgTsl3
+ return 0
+ wait_backup backup-minio-sharded
+ local backup_name=backup-minio-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-sharded to reach ready state......
+ check_backup_in_storage backup-minio-sharded minio rs0
+ local backup=backup-minio-sharded
+ local storage_type=minio
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=minio-service
++ get_backup_dest backup-minio-sharded
++ local backup_name=backup-minio-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-minio-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vnm2fZyN1F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kF12l4k8K5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vnm2fZyN1F
++ cat /tmp/tmp.kF12l4k8K5
++ rm /tmp/tmp.vnm2fZyN1F /tmp/tmp.kF12l4k8K5
++ return 0
+ backup_dest=operator-testing/2025-09-22T14:42:17Z
+ [[ minio == \m\i\n\i\o ]]
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-09-22T14:42:17Z/rs0/filelist.pbm
+ grep filelist.pbm
++ mktemp
+ local LAST_OUT=/tmp/tmp.F8OvWYAwVD
++ mktemp
+ local LAST_ERR=/tmp/tmp.IfPvpNqlJv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-09-22T14:42:17Z/rs0/filelist.pbm
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.F8OvWYAwVD
+ cat /tmp/tmp.IfPvpNqlJv
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-incremental-sharded-2770
+ rm /tmp/tmp.F8OvWYAwVD /tmp/tmp.IfPvpNqlJv
+ return 0
2025-09-22 14:42:25      11959 filelist.pbm
+ run_mongos 'use myApp\n db.test.insert({ x: 100502 })' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local 'command=use myApp\n db.test.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BBXfRVCTer
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RDk1wLOYge
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BBXfRVCTer
++ cat /tmp/tmp.RDk1wLOYge
++ rm /tmp/tmp.BBXfRVCTer /tmp/tmp.RDk1wLOYge
++ return 0
+ local client_container=psmdb-client-b9788d8bc-h62zq
+ kubectl_bin exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.GSDlBqUM5e
++ mktemp
+ local LAST_ERR=/tmp/tmp.RJBpSXEdqS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GSDlBqUM5e
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("088dd239-4b51-426c-b167-437c0d5bd35b") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.RJBpSXEdqS
+ rm /tmp/tmp.GSDlBqUM5e /tmp/tmp.RJBpSXEdqS
+ return 0
+ sleep 5
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770 -not-base-sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local postfix=-not-base-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770 mongodb '' '' 27017
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cziPc2OkpP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lzWoI9DVaW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cziPc2OkpP
++ cat /tmp/tmp.lzWoI9DVaW
++ rm /tmp/tmp.cziPc2OkpP /tmp/tmp.lzWoI9DVaW
++ return 0
+ local client_container=psmdb-client-b9788d8bc-h62zq
+ kubectl_bin exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.RokexgBtDB
++ mktemp
+ local LAST_ERR=/tmp/tmp.YQ67wdSVpT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RokexgBtDB
+ cat /tmp/tmp.YQ67wdSVpT
+ rm /tmp/tmp.RokexgBtDB /tmp/tmp.YQ67wdSVpT
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/find-not-base-sharded.json /tmp/tmp.Ui1CqFHLwe/find-not-base-sharded
+ backup_name_minio_not_base=backup-minio-not-base-sharded
+ run_backup minio backup-minio-not-base-sharded false
+ local storage=minio
+ local backup_name=backup-minio-not-base-sharded
+ local base=false
+ local backup_type=incremental
+ [[ false == \t\r\u\e ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-not-base-sharded/'
+ /usr/sbin/sed -e 's/storageName:/storageName: minio/'
+ yq '.spec.type="incremental"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.aS197iaZw6
++ mktemp
+ local LAST_ERR=/tmp/tmp.AJG7NyZgFv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aS197iaZw6
perconaservermongodbbackup.psmdb.percona.com/backup-minio-not-base-sharded created
+ cat /tmp/tmp.AJG7NyZgFv
+ rm /tmp/tmp.aS197iaZw6 /tmp/tmp.AJG7NyZgFv
+ return 0
+ wait_backup backup-minio-not-base-sharded
+ local backup_name=backup-minio-not-base-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-not-base-sharded to reach ready state........
+ check_backup_in_storage backup-minio-not-base-sharded minio rs0
+ local backup=backup-minio-not-base-sharded
+ local storage_type=minio
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=minio-service
++ get_backup_dest backup-minio-not-base-sharded
++ local backup_name=backup-minio-not-base-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-minio-not-base-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.aH4yujVa4A
+++ mktemp
++ local LAST_ERR=/tmp/tmp.86RQaUAmSP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-not-base-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aH4yujVa4A
++ cat /tmp/tmp.86RQaUAmSP
++ rm /tmp/tmp.aH4yujVa4A /tmp/tmp.86RQaUAmSP
++ return 0
+ backup_dest=operator-testing/2025-09-22T14:42:49Z
+ [[ minio == \m\i\n\i\o ]]
+ grep filelist.pbm
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-09-22T14:42:49Z/rs0/filelist.pbm
++ mktemp
+ local LAST_OUT=/tmp/tmp.8RI0BdSXDu
++ mktemp
+ local LAST_ERR=/tmp/tmp.veP1zS9yP4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-09-22T14:42:49Z/rs0/filelist.pbm
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8RI0BdSXDu
+ cat /tmp/tmp.veP1zS9yP4
+ rm /tmp/tmp.8RI0BdSXDu /tmp/tmp.veP1zS9yP4
+ return 0
2025-09-22 14:43:01      12759 filelist.pbm
+ '[' -z '' ']'
+ run_restore backup-aws-s3-sharded _restore_sharded
+ local backup_name=backup-aws-s3-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-09-22T14:43:11+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PmaQHJOfby
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rWZ8tmqI2U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PmaQHJOfby
++ cat /tmp/tmp.rWZ8tmqI2U
++ rm /tmp/tmp.PmaQHJOfby /tmp/tmp.rWZ8tmqI2U
++ return 0
+ local client_container=psmdb-client-b9788d8bc-h62zq
+ kubectl_bin exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.KFP1c2HjV1
++ mktemp
+ local LAST_ERR=/tmp/tmp.E2pUKclvNn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KFP1c2HjV1
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f807de41-ce8f-4a91-9c21-3ef2f87b3f4c") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.E2pUKclvNn
+ rm /tmp/tmp.KFP1c2HjV1 /tmp/tmp.E2pUKclvNn
+ return 0
+ log 'check backup and restore -- backup-aws-s3-sharded'
+ set +o xtrace
[2025-09-22T14:43:14+0000] check backup and restore -- backup-aws-s3-sharded
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3-sharded/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.pfT0K74hYl
++ mktemp
+ local LAST_ERR=/tmp/tmp.MV7pNv0bnY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pfT0K74hYl
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3-sharded created
+ cat /tmp/tmp.MV7pNv0bnY
+ rm /tmp/tmp.pfT0K74hYl /tmp/tmp.MV7pNv0bnY
+ return 0
+ run_recovery_check backup-aws-s3-sharded _restore_sharded
+ local backup_name=backup-aws-s3-sharded
+ local compare_suffix=_restore_sharded
+ local base=true
+ wait_restore backup-aws-s3-sharded some-name requested 0 3000
+ local backup_name=backup-aws-s3-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-aws-s3-sharded to reach state "requested" ....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-2770", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.FfLgGrGRZ2
++ mktemp
+ local LAST_ERR=/tmp/tmp.4ZZ8DzsOGU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FfLgGrGRZ2
+ cat /tmp/tmp.4ZZ8DzsOGU
+ rm /tmp/tmp.FfLgGrGRZ2 /tmp/tmp.4ZZ8DzsOGU
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ wait_restore backup-aws-s3-sharded some-name ready 0 3000
+ local backup_name=backup-aws-s3-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-aws-s3-sharded to reach state "ready" ..OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.x0E3iKtsw3
++ mktemp
+ local LAST_ERR=/tmp/tmp.bpxY8V16mO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.x0E3iKtsw3
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-incremental-sharded-2770"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-server-mongodb-operator:main-backup","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-incremental-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"minio":{"main":true,"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
    percona.com/resync-pbm: "true"
  creationTimestamp: "2025-09-22T14:36:06Z"
  generation: 2
  name: some-name
  namespace: demand-backup-incremental-sharded-2770
  resourceVersion: "1758552600172815013"
  uid: 96ae758a-84eb-40a8-bdcc-54287519520b
spec:
  backup:
    enabled: true
    image: perconalab/percona-server-mongodb-operator:main-backup
    storages:
      aws-s3:
        s3:
          bucket: operator-testing
          credentialsSecret: aws-s3-secret
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      azure-blob:
        azure:
          container: operator-testing
          credentialsSecret: azure-secret
          prefix: psmdb-demand-backup-incremental-sharded
        type: azure
      gcp-cs:
        s3:
          bucket: operator-testing
          credentialsSecret: gcp-cs-secret
          endpointUrl: https://storage.googleapis.com
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      minio:
        main: true
        s3:
          bucket: operator-testing
          credentialsSecret: minio-secret
          endpointUrl: http://minio-service:9000/
          insecureSkipTLSVerify: false
          region: us-east-1
        type: s3
    tasks:
    - compressionType: gzip
      enabled: true
      name: weekly
      schedule: 0 0 * * 0
      storageName: aws-s3
  crVersion: 1.21.0
  image: perconalab/percona-server-mongodb-operator:main-mongod7.0
  imagePullPolicy: Always
  replsets:
  - affinity:
      antiAffinityTopologyKey: none
    configuration: |
      operationProfiling:
        mode: slowOp
        slowOpThresholdMs: 100
      security:
        enableEncryption: true
        redactClientLogData: false
      setParameter:
        ttlMonitorSleepSecs: 60
        wiredTigerConcurrentReadTransactions: 128
        wiredTigerConcurrentWriteTransactions: 128
      storage:
        engine: wiredTiger
        wiredTiger:
          collectionConfig:
            blockCompressor: snappy
          engineConfig:
            directoryForIndexes: false
            journalCompressor: snappy
          indexConfig:
            prefixCompression: true
    expose:
      enabled: false
      type: ClusterIP
    name: rs0
    resources:
      limits:
        cpu: 500m
        memory: 1G
      requests:
        cpu: 100m
        memory: 0.1G
    size: 3
    volumeSpec:
      persistentVolumeClaim:
        resources:
          requests:
            storage: 3Gi
  secrets:
    users: some-users
  sharding:
    configsvrReplSet:
      affinity:
        antiAffinityTopologyKey: none
      configuration: |
        operationProfiling:
          mode: slowOp
          slowOpThresholdMs: 100
        security:
          enableEncryption: true
          redactClientLogData: false
        setParameter:
          ttlMonitorSleepSecs: 60
          wiredTigerConcurrentReadTransactions: 128
          wiredTigerConcurrentWriteTransactions: 128
        storage:
          engine: wiredTiger
          wiredTiger:
            collectionConfig:
              blockCompressor: snappy
            engineConfig:
              directoryForIndexes: false
              journalCompressor: snappy
            indexConfig:
              prefixCompression: true
      expose:
        enabled: false
        type: ClusterIP
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
      volumeSpec:
        persistentVolumeClaim:
          resources:
            requests:
              storage: 3Gi
    enabled: true
    mongos:
      affinity:
        antiAffinityTopologyKey: none
      expose:
        type: LoadBalancer
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
  upgradeOptions:
    apply: Never
status:
  backupConfigHash: 657556472134be8a3e4b574eeba7d6db2138970454efb713e0add6290c72d12f
  conditions:
  - lastTransitionTime: "2025-09-22T14:36:06Z"
    status: "True"
    type: sharding
  - lastTransitionTime: "2025-09-22T14:36:10Z"
    status: "True"
    type: initializing
  - lastTransitionTime: "2025-09-22T14:37:31Z"
    message: 'rs0: ready'
    reason: RSReady
    status: "True"
    type: ready
  host: 35.239.64.210
  mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
  mongoVersion: 7.0.24-13
  mongos:
    ready: 0
    size: 0
    status: initializing
  observedGeneration: 2
  ready: 6
  replsets:
    cfg:
      initialized: true
      ready: 3
      size: 3
      status: ready
    rs0:
      added_as_shard: true
      initialized: true
      ready: 3
      size: 3
      status: ready
  size: 6
  state: initializing
+ cat /tmp/tmp.bpxY8V16mO
+ rm /tmp/tmp.x0E3iKtsw3 /tmp/tmp.bpxY8V16mO
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NKxtIXXg6M
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FxMR702wt3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NKxtIXXg6M
++ cat /tmp/tmp.FxMR702wt3
++ rm /tmp/tmp.NKxtIXXg6M /tmp/tmp.FxMR702wt3
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 60
+ local cluster_name=some-name
+ local wait_time=60
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.A7VzJp0AuW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xOXRP2Gaah
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.A7VzJp0AuW
++ cat /tmp/tmp.xOXRP2Gaah
++ rm /tmp/tmp.A7VzJp0AuW /tmp/tmp.xOXRP2Gaah
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ICV4sP2tab
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oaSK6xiUbe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ICV4sP2tab
++ cat /tmp/tmp.oaSK6xiUbe
++ rm /tmp/tmp.ICV4sP2tab /tmp/tmp.oaSK6xiUbe
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oVRhIUlKCF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oEOmwB2Vqp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oVRhIUlKCF
++ cat /tmp/tmp.oEOmwB2Vqp
++ rm /tmp/tmp.oVRhIUlKCF /tmp/tmp.oEOmwB2Vqp
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LvNVK1ynRP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gRmxffaBSr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LvNVK1ynRP
++ cat /tmp/tmp.gRmxffaBSr
++ rm /tmp/tmp.LvNVK1ynRP /tmp/tmp.gRmxffaBSr
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yVJ3bUMbcf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2cfCNtxM1f
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yVJ3bUMbcf
++ cat /tmp/tmp.2cfCNtxM1f
++ rm /tmp/tmp.yVJ3bUMbcf /tmp/tmp.2cfCNtxM1f
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.j7SF3eg8Fu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GuJpZzA5cu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.j7SF3eg8Fu
++ cat /tmp/tmp.GuJpZzA5cu
++ rm /tmp/tmp.j7SF3eg8Fu /tmp/tmp.GuJpZzA5cu
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a2rTiteRKx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KzPMh04ggb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.a2rTiteRKx
++ cat /tmp/tmp.KzPMh04ggb
++ rm /tmp/tmp.a2rTiteRKx /tmp/tmp.KzPMh04ggb
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tzfn5HHj1J
+++ mktemp
++ local LAST_ERR=/tmp/tmp.on1E7rnjGg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tzfn5HHj1J
++ cat /tmp/tmp.on1E7rnjGg
++ rm /tmp/tmp.tzfn5HHj1J /tmp/tmp.on1E7rnjGg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.74ydPDpqU8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wSLj8uWTkR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.74ydPDpqU8
++ cat /tmp/tmp.wSLj8uWTkR
++ rm /tmp/tmp.74ydPDpqU8 /tmp/tmp.wSLj8uWTkR
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ostnsdVyEr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PFW1G1ojJP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ostnsdVyEr
++ cat /tmp/tmp.PFW1G1ojJP
++ rm /tmp/tmp.ostnsdVyEr /tmp/tmp.PFW1G1ojJP
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nX1wkCv8VO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cY7T5TTF08
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nX1wkCv8VO
++ cat /tmp/tmp.cY7T5TTF08
++ rm /tmp/tmp.nX1wkCv8VO /tmp/tmp.cY7T5TTF08
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NzLdTlw4QR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IkOlbILMiN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NzLdTlw4QR
++ cat /tmp/tmp.IkOlbILMiN
++ rm /tmp/tmp.NzLdTlw4QR /tmp/tmp.IkOlbILMiN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TcMPRJTzqi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Jx2KnQzy0l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TcMPRJTzqi
++ cat /tmp/tmp.Jx2KnQzy0l
++ rm /tmp/tmp.TcMPRJTzqi /tmp/tmp.Jx2KnQzy0l
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BU26FXTnmg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bfx3NYlaWL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BU26FXTnmg
++ cat /tmp/tmp.bfx3NYlaWL
++ rm /tmp/tmp.BU26FXTnmg /tmp/tmp.bfx3NYlaWL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 14 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aAfk66BTjW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dIZBRSXjlj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aAfk66BTjW
++ cat /tmp/tmp.dIZBRSXjlj
++ rm /tmp/tmp.aAfk66BTjW /tmp/tmp.dIZBRSXjlj
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 15 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wYVp5CC4IT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Sfaz7XxGIu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wYVp5CC4IT
++ cat /tmp/tmp.Sfaz7XxGIu
++ rm /tmp/tmp.wYVp5CC4IT /tmp/tmp.Sfaz7XxGIu
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 16 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yW32oRhqEb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xch8PCAHAe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yW32oRhqEb
++ cat /tmp/tmp.xch8PCAHAe
++ rm /tmp/tmp.yW32oRhqEb /tmp/tmp.xch8PCAHAe
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish
+ [[ true == true ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XwcSJeOdKZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xw3bBzAdhp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XwcSJeOdKZ
++ cat /tmp/tmp.xw3bBzAdhp
++ rm /tmp/tmp.XwcSJeOdKZ /tmp/tmp.xw3bBzAdhp
++ return 0
+ local client_container=psmdb-client-b9788d8bc-h62zq
+ kubectl_bin exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0JmlseTj0C
++ mktemp
+ local LAST_ERR=/tmp/tmp.RmxeINMhnL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0JmlseTj0C
+ cat /tmp/tmp.RmxeINMhnL
+ rm /tmp/tmp.0JmlseTj0C /tmp/tmp.RmxeINMhnL
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.Ui1CqFHLwe/find-sharded
+ check_exported_mongos_service_endpoint 35.239.64.210
+ local host=35.239.64.210
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0AAk6waaO0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T0XgLvpyau
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0AAk6waaO0
++ cat /tmp/tmp.T0XgLvpyau
++ rm /tmp/tmp.0AAk6waaO0 /tmp/tmp.T0XgLvpyau
++ return 0
+ '[' 35.239.64.210 '!=' 35.239.64.210 ']'
+ run_restore backup-gcp-cs-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-09-22T14:53:16+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jeUl365fuM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WsZjS6gBee
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jeUl365fuM
++ cat /tmp/tmp.WsZjS6gBee
++ rm /tmp/tmp.jeUl365fuM /tmp/tmp.WsZjS6gBee
++ return 0
+ local client_container=psmdb-client-b9788d8bc-h62zq
+ kubectl_bin exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9SFqst6Rtm
++ mktemp
+ local LAST_ERR=/tmp/tmp.CLtwOssEqs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9SFqst6Rtm
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("a80f3b12-7e5f-4a7f-9049-298c4c4d1d2a") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.CLtwOssEqs
+ rm /tmp/tmp.9SFqst6Rtm /tmp/tmp.CLtwOssEqs
+ return 0
+ log 'check backup and restore -- backup-gcp-cs-sharded'
+ set +o xtrace
[2025-09-22T14:53:20+0000] check backup and restore -- backup-gcp-cs-sharded
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sharded/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.qYZCLLO5XZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.WbUYQZOdc3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qYZCLLO5XZ
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sharded created
+ cat /tmp/tmp.WbUYQZOdc3
+ rm /tmp/tmp.qYZCLLO5XZ /tmp/tmp.WbUYQZOdc3
+ return 0
+ run_recovery_check backup-gcp-cs-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sharded
+ local compare_suffix=_restore_sharded
+ local base=true
+ wait_restore backup-gcp-cs-sharded some-name requested 0 3000
+ local backup_name=backup-gcp-cs-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-gcp-cs-sharded to reach state "requested" ....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-2770", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.v4bg5Rjiml
++ mktemp
+ local LAST_ERR=/tmp/tmp.jbHxLMOeDn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.v4bg5Rjiml
+ cat /tmp/tmp.jbHxLMOeDn
+ rm /tmp/tmp.v4bg5Rjiml /tmp/tmp.jbHxLMOeDn
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ wait_restore backup-gcp-cs-sharded some-name ready 0 3000
+ local backup_name=backup-gcp-cs-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-gcp-cs-sharded to reach state "ready" ..OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.nk0heJIh5W
++ mktemp
+ local LAST_ERR=/tmp/tmp.0wbwr6CGwh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nk0heJIh5W
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-incremental-sharded-2770"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-server-mongodb-operator:main-backup","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-incremental-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"minio":{"main":true,"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
    percona.com/resync-pbm: "true"
  creationTimestamp: "2025-09-22T14:36:06Z"
  generation: 2
  name: some-name
  namespace: demand-backup-incremental-sharded-2770
  resourceVersion: "1758553216311343013"
  uid: 96ae758a-84eb-40a8-bdcc-54287519520b
spec:
  backup:
    enabled: true
    image: perconalab/percona-server-mongodb-operator:main-backup
    storages:
      aws-s3:
        s3:
          bucket: operator-testing
          credentialsSecret: aws-s3-secret
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      azure-blob:
        azure:
          container: operator-testing
          credentialsSecret: azure-secret
          prefix: psmdb-demand-backup-incremental-sharded
        type: azure
      gcp-cs:
        s3:
          bucket: operator-testing
          credentialsSecret: gcp-cs-secret
          endpointUrl: https://storage.googleapis.com
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      minio:
        main: true
        s3:
          bucket: operator-testing
          credentialsSecret: minio-secret
          endpointUrl: http://minio-service:9000/
          insecureSkipTLSVerify: false
          region: us-east-1
        type: s3
    tasks:
    - compressionType: gzip
      enabled: true
      name: weekly
      schedule: 0 0 * * 0
      storageName: aws-s3
  crVersion: 1.21.0
  image: perconalab/percona-server-mongodb-operator:main-mongod7.0
  imagePullPolicy: Always
  replsets:
  - affinity:
      antiAffinityTopologyKey: none
    configuration: |
      operationProfiling:
        mode: slowOp
        slowOpThresholdMs: 100
      security:
        enableEncryption: true
        redactClientLogData: false
      setParameter:
        ttlMonitorSleepSecs: 60
        wiredTigerConcurrentReadTransactions: 128
        wiredTigerConcurrentWriteTransactions: 128
      storage:
        engine: wiredTiger
        wiredTiger:
          collectionConfig:
            blockCompressor: snappy
          engineConfig:
            directoryForIndexes: false
            journalCompressor: snappy
          indexConfig:
            prefixCompression: true
    expose:
      enabled: false
      type: ClusterIP
    name: rs0
    resources:
      limits:
        cpu: 500m
        memory: 1G
      requests:
        cpu: 100m
        memory: 0.1G
    size: 3
    volumeSpec:
      persistentVolumeClaim:
        resources:
          requests:
            storage: 3Gi
  secrets:
    users: some-users
  sharding:
    configsvrReplSet:
      affinity:
        antiAffinityTopologyKey: none
      configuration: |
        operationProfiling:
          mode: slowOp
          slowOpThresholdMs: 100
        security:
          enableEncryption: true
          redactClientLogData: false
        setParameter:
          ttlMonitorSleepSecs: 60
          wiredTigerConcurrentReadTransactions: 128
          wiredTigerConcurrentWriteTransactions: 128
        storage:
          engine: wiredTiger
          wiredTiger:
            collectionConfig:
              blockCompressor: snappy
            engineConfig:
              directoryForIndexes: false
              journalCompressor: snappy
            indexConfig:
              prefixCompression: true
      expose:
        enabled: false
        type: ClusterIP
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
      volumeSpec:
        persistentVolumeClaim:
          resources:
            requests:
              storage: 3Gi
    enabled: true
    mongos:
      affinity:
        antiAffinityTopologyKey: none
      expose:
        type: LoadBalancer
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
  upgradeOptions:
    apply: Never
status:
  backupConfigHash: 657556472134be8a3e4b574eeba7d6db2138970454efb713e0add6290c72d12f
  conditions:
  - lastTransitionTime: "2025-09-22T14:36:06Z"
    status: "True"
    type: sharding
  - lastTransitionTime: "2025-09-22T14:36:10Z"
    status: "True"
    type: initializing
  - lastTransitionTime: "2025-09-22T14:37:31Z"
    message: 'rs0: ready'
    reason: RSReady
    status: "True"
    type: ready
  - lastTransitionTime: "2025-09-22T14:50:42Z"
    message: |-
      handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-2770.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-2770.svc.cluster.local on 34.118.224.10:53: no such host }, ] }
      handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-2770.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-2770.svc.cluster.local on 34.118.224.10:53: no such host }, ] }
    reason: ErrorReconcile
    status: "True"
    type: error
  host: 35.239.64.210
  mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
  mongoVersion: 7.0.24-13
  mongos:
    ready: 0
    size: 0
    status: initializing
  observedGeneration: 2
  ready: 6
  replsets:
    cfg:
      initialized: true
      ready: 3
      size: 3
      status: ready
    rs0:
      added_as_shard: true
      initialized: true
      ready: 3
      size: 3
      status: ready
  size: 6
  state: initializing
+ cat /tmp/tmp.0wbwr6CGwh
+ rm /tmp/tmp.nk0heJIh5W /tmp/tmp.0wbwr6CGwh
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zR7Z8uX5nH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uPHZqP1kA2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zR7Z8uX5nH
++ cat /tmp/tmp.uPHZqP1kA2
++ rm /tmp/tmp.zR7Z8uX5nH /tmp/tmp.uPHZqP1kA2
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 60
+ local cluster_name=some-name
+ local wait_time=60
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P7zwXyUtAc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HLoZ7AaRJY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.P7zwXyUtAc
++ cat /tmp/tmp.HLoZ7AaRJY
++ rm /tmp/tmp.P7zwXyUtAc /tmp/tmp.HLoZ7AaRJY
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FpeVqWXa7f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cMOBAuwRPl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FpeVqWXa7f
++ cat /tmp/tmp.cMOBAuwRPl
++ rm /tmp/tmp.FpeVqWXa7f /tmp/tmp.cMOBAuwRPl
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eNicwCARIe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QMKExv04xQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eNicwCARIe
++ cat /tmp/tmp.QMKExv04xQ
++ rm /tmp/tmp.eNicwCARIe /tmp/tmp.QMKExv04xQ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DcQUwSeHDy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OQ4hBk2wPL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DcQUwSeHDy
++ cat /tmp/tmp.OQ4hBk2wPL
++ rm /tmp/tmp.DcQUwSeHDy /tmp/tmp.OQ4hBk2wPL
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xDj9H3qVB3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NJye4AjEvn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xDj9H3qVB3
++ cat /tmp/tmp.NJye4AjEvn
++ rm /tmp/tmp.xDj9H3qVB3 /tmp/tmp.NJye4AjEvn
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bfrdpA88e4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DSDHLJlOVZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bfrdpA88e4
++ cat /tmp/tmp.DSDHLJlOVZ
++ rm /tmp/tmp.bfrdpA88e4 /tmp/tmp.DSDHLJlOVZ
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Rd2xO9bEIJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T3RTrAyJKX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Rd2xO9bEIJ
++ cat /tmp/tmp.T3RTrAyJKX
++ rm /tmp/tmp.Rd2xO9bEIJ /tmp/tmp.T3RTrAyJKX
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZRDebLfRhT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LPlVcktgMz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZRDebLfRhT
++ cat /tmp/tmp.LPlVcktgMz
++ rm /tmp/tmp.ZRDebLfRhT /tmp/tmp.LPlVcktgMz
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hBAliOQGu2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N67kRZk8I3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hBAliOQGu2
++ cat /tmp/tmp.N67kRZk8I3
++ rm /tmp/tmp.hBAliOQGu2 /tmp/tmp.N67kRZk8I3
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yVI2e2MJO7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9AHIbDKWH3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yVI2e2MJO7
++ cat /tmp/tmp.9AHIbDKWH3
++ rm /tmp/tmp.yVI2e2MJO7 /tmp/tmp.9AHIbDKWH3
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YKi2nbeOJj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dHqET2fEKO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YKi2nbeOJj
++ cat /tmp/tmp.dHqET2fEKO
++ rm /tmp/tmp.YKi2nbeOJj /tmp/tmp.dHqET2fEKO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.m54oKZzaeC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SPkueXDwiq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.m54oKZzaeC
++ cat /tmp/tmp.SPkueXDwiq
++ rm /tmp/tmp.m54oKZzaeC /tmp/tmp.SPkueXDwiq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dTjJgRfTQd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Xa8gbmttzs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dTjJgRfTQd
++ cat /tmp/tmp.Xa8gbmttzs
++ rm /tmp/tmp.dTjJgRfTQd /tmp/tmp.Xa8gbmttzs
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NGdzEuj7V3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gZxcu9B8SS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NGdzEuj7V3
++ cat /tmp/tmp.gZxcu9B8SS
++ rm /tmp/tmp.NGdzEuj7V3 /tmp/tmp.gZxcu9B8SS
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish
+ [[ true == true ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xXiWdYnXms
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TxnmoFCYMd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xXiWdYnXms
++ cat /tmp/tmp.TxnmoFCYMd
++ rm /tmp/tmp.xXiWdYnXms /tmp/tmp.TxnmoFCYMd
++ return 0
+ local client_container=psmdb-client-b9788d8bc-h62zq
+ kubectl_bin exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.q2dfb9h3g9
++ mktemp
+ local LAST_ERR=/tmp/tmp.giKs5FCbLt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q2dfb9h3g9
+ cat /tmp/tmp.giKs5FCbLt
+ rm /tmp/tmp.q2dfb9h3g9 /tmp/tmp.giKs5FCbLt
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.Ui1CqFHLwe/find-sharded
+ check_exported_mongos_service_endpoint 35.239.64.210
+ local host=35.239.64.210
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zRgKXpbejh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s6nMaReBiV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zRgKXpbejh
++ cat /tmp/tmp.s6nMaReBiV
++ rm /tmp/tmp.zRgKXpbejh /tmp/tmp.s6nMaReBiV
++ return 0
+ '[' 35.239.64.210 '!=' 35.239.64.210 ']'
+ run_restore backup-azure-blob-sharded _restore_sharded
+ local backup_name=backup-azure-blob-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-09-22T15:02:58+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hPsVFHxWxZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Dn9gNdA6IX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hPsVFHxWxZ
++ cat /tmp/tmp.Dn9gNdA6IX
++ rm /tmp/tmp.hPsVFHxWxZ /tmp/tmp.Dn9gNdA6IX
++ return 0
+ local client_container=psmdb-client-b9788d8bc-h62zq
+ kubectl_bin exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7MMLrlsJ96
++ mktemp
+ local LAST_ERR=/tmp/tmp.8v9i5Ni2AD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-h62zq -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7MMLrlsJ96
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-2770.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("126a4395-f734-417e-9e31-b2d6777df85f") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.8v9i5Ni2AD
+ rm /tmp/tmp.7MMLrlsJ96 /tmp/tmp.8v9i5Ni2AD
+ return 0
+ log 'check backup and restore -- backup-azure-blob-sharded'
+ set +o xtrace
[2025-09-22T15:03:01+0000] check backup and restore -- backup-azure-blob-sharded
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob-sharded/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.tG49wPKiLE
++ mktemp
+ local LAST_ERR=/tmp/tmp.2dKtFFgw4E
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tG49wPKiLE
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob-sharded created
+ cat /tmp/tmp.2dKtFFgw4E
+ rm /tmp/tmp.tG49wPKiLE /tmp/tmp.2dKtFFgw4E
+ return 0
+ run_recovery_check backup-azure-blob-sharded _restore_sharded
+ local backup_name=backup-azure-blob-sharded
+ local compare_suffix=_restore_sharded
+ local base=true
+ wait_restore backup-azure-blob-sharded some-name requested 0 3000
+ local backup_name=backup-azure-blob-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-azure-blob-sharded to reach state "requested" ...OK after 3 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-2770", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.E6NP1U2CC9
++ mktemp
+ local LAST_ERR=/tmp/tmp.VakLpY6rqm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E6NP1U2CC9
+ cat /tmp/tmp.VakLpY6rqm
+ rm /tmp/tmp.E6NP1U2CC9 /tmp/tmp.VakLpY6rqm
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.Ui1CqFHLwe/statefulset_some-name-rs0.yml
+ wait_restore backup-azure-blob-sharded some-name ready 0 3000
+ local backup_name=backup-azure-blob-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-azure-blob-sharded to reach state "ready" ....2025-09-22T15:10:02.416Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "40b7ffb2-ae18-4503-a1fb-d4b81c7e127f", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:10:02.417Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "40b7ffb2-ae18-4503-a1fb-d4b81c7e127f", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:10:02.779Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "40b7ffb2-ae18-4503-a1fb-d4b81c7e127f", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"cfg","start_ts":0,"status":"starting","size":0,"size_uncompressed":0,"last_transition_ts":1758553628,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-09-22T15:10:07.780Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8a82b3ec-9edc-45a5-96e5-5e8b20c10b4f", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-2770"}
2025-09-22T15:10:07.781Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8a82b3ec-9edc-45a5-96e5-5e8b20c10b4f", "pod": "some-name-rs0-0"}
2025-09-22T15:10:07.781Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8a82b3ec-9edc-45a5-96e5-5e8b20c10b4f", "pod": "some-name-rs0-2"}
2025-09-22T15:10:07.781Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8a82b3ec-9edc-45a5-96e5-5e8b20c10b4f", "pod": "some-name-rs0-1"}
2025-09-22T15:10:07.781Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8a82b3ec-9edc-45a5-96e5-5e8b20c10b4f", "sts": "some-name-rs0", "replset": "rs0"}
2025-09-22T15:10:07.781Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8a82b3ec-9edc-45a5-96e5-5e8b20c10b4f", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:10:07.781Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8a82b3ec-9edc-45a5-96e5-5e8b20c10b4f", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:10:07.979Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8a82b3ec-9edc-45a5-96e5-5e8b20c10b4f", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"cfg","start_ts":0,"status":"starting","size":0,"size_uncompressed":0,"last_transition_ts":1758553628,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-09-22T15:10:12.980Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "88981f70-a2ca-42bf-9562-637bd778ac8c", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-2770"}
2025-09-22T15:10:12.981Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "88981f70-a2ca-42bf-9562-637bd778ac8c", "pod": "some-name-rs0-1"}
2025-09-22T15:10:12.981Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "88981f70-a2ca-42bf-9562-637bd778ac8c", "pod": "some-name-rs0-0"}
2025-09-22T15:10:12.981Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "88981f70-a2ca-42bf-9562-637bd778ac8c", "pod": "some-name-rs0-2"}
2025-09-22T15:10:12.981Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "88981f70-a2ca-42bf-9562-637bd778ac8c", "sts": "some-name-rs0", "replset": "rs0"}
2025-09-22T15:10:12.981Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "88981f70-a2ca-42bf-9562-637bd778ac8c", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:10:12.981Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "88981f70-a2ca-42bf-9562-637bd778ac8c", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:10:13.351Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "88981f70-a2ca-42bf-9562-637bd778ac8c", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"cfg","start_ts":0,"status":"starting","size":0,"size_uncompressed":0,"last_transition_ts":1758553628,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-09-22T15:10:18.352Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "5b39fc2f-380b-4a51-8ebd-e43ffcc5c82c", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-2770"}
2025-09-22T15:10:18.353Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "5b39fc2f-380b-4a51-8ebd-e43ffcc5c82c", "pod": "some-name-rs0-0"}
2025-09-22T15:10:18.353Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "5b39fc2f-380b-4a51-8ebd-e43ffcc5c82c", "pod": "some-name-rs0-2"}
2025-09-22T15:10:18.353Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "5b39fc2f-380b-4a51-8ebd-e43ffcc5c82c", "pod": "some-name-rs0-1"}
2025-09-22T15:10:18.353Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "5b39fc2f-380b-4a51-8ebd-e43ffcc5c82c", "sts": "some-name-rs0", "replset": "rs0"}
2025-09-22T15:10:18.353Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "5b39fc2f-380b-4a51-8ebd-e43ffcc5c82c", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:10:18.353Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "5b39fc2f-380b-4a51-8ebd-e43ffcc5c82c", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:10:18.790Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "5b39fc2f-380b-4a51-8ebd-e43ffcc5c82c", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"cfg","start_ts":0,"status":"starting","size":0,"size_uncompressed":0,"last_transition_ts":1758553628,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-09-22T15:10:23.791Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8cc9d031-6942-458a-86e3-745cce1b822d", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-2770"}
2025-09-22T15:10:23.793Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8cc9d031-6942-458a-86e3-745cce1b822d", "pod": "some-name-rs0-2"}
2025-09-22T15:10:23.793Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8cc9d031-6942-458a-86e3-745cce1b822d", "pod": "some-name-rs0-1"}
2025-09-22T15:10:23.793Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8cc9d031-6942-458a-86e3-745cce1b822d", "pod": "some-name-rs0-0"}
2025-09-22T15:10:23.793Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8cc9d031-6942-458a-86e3-745cce1b822d", "sts": "some-name-rs0", "replset": "rs0"}
2025-09-22T15:10:23.793Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8cc9d031-6942-458a-86e3-745cce1b822d", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:10:23.793Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8cc9d031-6942-458a-86e3-745cce1b822d", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:10:24.422Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "8cc9d031-6942-458a-86e3-745cce1b822d", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"starting","size":0,"size_uncompressed":0,"last_transition_ts":1758553628,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-09-22T15:10:29.423Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "3d2a7b85-e0af-4689-9b3b-78c0f9ad6e93", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-2770"}
2025-09-22T15:10:29.425Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "3d2a7b85-e0af-4689-9b3b-78c0f9ad6e93", "pod": "some-name-rs0-1"}
2025-09-22T15:10:29.425Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "3d2a7b85-e0af-4689-9b3b-78c0f9ad6e93", "pod": "some-name-rs0-0"}
2025-09-22T15:10:29.425Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "3d2a7b85-e0af-4689-9b3b-78c0f9ad6e93", "pod": "some-name-rs0-2"}
2025-09-22T15:10:29.425Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "3d2a7b85-e0af-4689-9b3b-78c0f9ad6e93", "sts": "some-name-rs0", "replset": "rs0"}
2025-09-22T15:10:29.425Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "3d2a7b85-e0af-4689-9b3b-78c0f9ad6e93", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:10:29.425Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "3d2a7b85-e0af-4689-9b3b-78c0f9ad6e93", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:10:29.638Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "3d2a7b85-e0af-4689-9b3b-78c0f9ad6e93", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"cfg","start_ts":0,"status":"starting","size":0,"size_uncompressed":0,"last_transition_ts":1758553628,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-09-22T15:10:34.639Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "67bf115a-92d8-40a8-9f29-390085faef79", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-2770"}
2025-09-22T15:10:34.641Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "67bf115a-92d8-40a8-9f29-390085faef79", "pod": "some-name-rs0-0"}
2025-09-22T15:10:34.641Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "67bf115a-92d8-40a8-9f29-390085faef79", "pod": "some-name-rs0-2"}
2025-09-22T15:10:34.641Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "67bf115a-92d8-40a8-9f29-390085faef79", "pod": "some-name-rs0-1"}
2025-09-22T15:10:34.641Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "67bf115a-92d8-40a8-9f29-390085faef79", "sts": "some-name-rs0", "replset": "rs0"}
2025-09-22T15:10:34.641Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "67bf115a-92d8-40a8-9f29-390085faef79", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:10:34.641Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "67bf115a-92d8-40a8-9f29-390085faef79", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:10:35.012Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "67bf115a-92d8-40a8-9f29-390085faef79", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"starting","size":0,"size_uncompressed":0,"last_transition_ts":1758553628,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-09-22T15:10:40.013Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "b3a8ce32-c40e-43f9-a842-ebcc6e3b32d4", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-2770"}
2025-09-22T15:10:40.014Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "b3a8ce32-c40e-43f9-a842-ebcc6e3b32d4", "pod": "some-name-rs0-2"}
2025-09-22T15:10:40.014Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "b3a8ce32-c40e-43f9-a842-ebcc6e3b32d4", "pod": "some-name-rs0-1"}
2025-09-22T15:10:40.014Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "b3a8ce32-c40e-43f9-a842-ebcc6e3b32d4", "pod": "some-name-rs0-0"}
2025-09-22T15:10:40.014Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "b3a8ce32-c40e-43f9-a842-ebcc6e3b32d4", "sts": "some-name-rs0", "replset": "rs0"}
2025-09-22T15:10:40.014Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "b3a8ce32-c40e-43f9-a842-ebcc6e3b32d4", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:10:40.014Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "b3a8ce32-c40e-43f9-a842-ebcc6e3b32d4", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:10:40.399Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "b3a8ce32-c40e-43f9-a842-ebcc6e3b32d4", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"cfg","start_ts":0,"status":"starting","size":0,"size_uncompressed":0,"last_transition_ts":1758553628,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-09-22T15:10:45.399Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "cd93c221-5b0e-4165-b763-c0ff5dc66c20", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-2770"}
2025-09-22T15:10:45.401Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "cd93c221-5b0e-4165-b763-c0ff5dc66c20", "pod": "some-name-rs0-2"}
2025-09-22T15:10:45.401Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "cd93c221-5b0e-4165-b763-c0ff5dc66c20", "pod": "some-name-rs0-1"}
2025-09-22T15:10:45.401Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "cd93c221-5b0e-4165-b763-c0ff5dc66c20", "pod": "some-name-rs0-0"}
2025-09-22T15:10:45.401Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "cd93c221-5b0e-4165-b763-c0ff5dc66c20", "sts": "some-name-rs0", "replset": "rs0"}
2025-09-22T15:10:45.401Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "cd93c221-5b0e-4165-b763-c0ff5dc66c20", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:10:45.401Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "cd93c221-5b0e-4165-b763-c0ff5dc66c20", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:10:45.796Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "cd93c221-5b0e-4165-b763-c0ff5dc66c20", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"cfg","start_ts":0,"status":"starting","size":0,"size_uncompressed":0,"last_transition_ts":1758553628,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-09-22T15:10:50.797Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "6a953ecf-d6ae-441e-8761-d62018214616", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-2770"}
2025-09-22T15:10:50.798Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "6a953ecf-d6ae-441e-8761-d62018214616", "pod": "some-name-rs0-0"}
2025-09-22T15:10:50.798Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "6a953ecf-d6ae-441e-8761-d62018214616", "pod": "some-name-rs0-2"}
2025-09-22T15:10:50.798Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "6a953ecf-d6ae-441e-8761-d62018214616", "pod": "some-name-rs0-1"}
2025-09-22T15:10:50.798Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "6a953ecf-d6ae-441e-8761-d62018214616", "sts": "some-name-rs0", "replset": "rs0"}
2025-09-22T15:10:50.798Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "6a953ecf-d6ae-441e-8761-d62018214616", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:10:50.798Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "6a953ecf-d6ae-441e-8761-d62018214616", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:10:51.181Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "6a953ecf-d6ae-441e-8761-d62018214616", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"cfg","start_ts":0,"status":"starting","size":0,"size_uncompressed":0,"last_transition_ts":1758553628,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-09-22T15:10:56.182Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "9f291ad7-62f0-4494-ad70-ba13f99cbc5f", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-2770"}
2025-09-22T15:10:56.183Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "9f291ad7-62f0-4494-ad70-ba13f99cbc5f", "pod": "some-name-rs0-0"}
2025-09-22T15:10:56.183Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "9f291ad7-62f0-4494-ad70-ba13f99cbc5f", "pod": "some-name-rs0-2"}
2025-09-22T15:10:56.183Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "9f291ad7-62f0-4494-ad70-ba13f99cbc5f", "pod": "some-name-rs0-1"}
2025-09-22T15:10:56.183Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "9f291ad7-62f0-4494-ad70-ba13f99cbc5f", "sts": "some-name-rs0", "replset": "rs0"}
2025-09-22T15:10:56.183Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "9f291ad7-62f0-4494-ad70-ba13f99cbc5f", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:10:56.183Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "9f291ad7-62f0-4494-ad70-ba13f99cbc5f", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:10:56.634Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "9f291ad7-62f0-4494-ad70-ba13f99cbc5f", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"starting","size":0,"size_uncompressed":0,"last_transition_ts":1758553628,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-09-22T15:11:01.635Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "26f3fbde-d432-479a-816b-8c4e6dcd4877", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-2770"}
2025-09-22T15:11:01.636Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "26f3fbde-d432-479a-816b-8c4e6dcd4877", "pod": "some-name-rs0-1"}
2025-09-22T15:11:01.636Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "26f3fbde-d432-479a-816b-8c4e6dcd4877", "pod": "some-name-rs0-0"}
2025-09-22T15:11:01.636Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "26f3fbde-d432-479a-816b-8c4e6dcd4877", "pod": "some-name-rs0-2"}
2025-09-22T15:11:01.636Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "26f3fbde-d432-479a-816b-8c4e6dcd4877", "sts": "some-name-rs0", "replset": "rs0"}
2025-09-22T15:11:01.636Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "26f3fbde-d432-479a-816b-8c4e6dcd4877", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:11:01.636Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "26f3fbde-d432-479a-816b-8c4e6dcd4877", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:11:02.011Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "26f3fbde-d432-479a-816b-8c4e6dcd4877", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"starting","size":0,"size_uncompressed":0,"last_transition_ts":1758553628,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-09-22T15:11:07.012Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "99bfa511-0e8e-485e-a8b6-f971c9ba2919", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-2770"}
2025-09-22T15:11:07.013Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "99bfa511-0e8e-485e-a8b6-f971c9ba2919", "pod": "some-name-rs0-1"}
2025-09-22T15:11:07.013Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "99bfa511-0e8e-485e-a8b6-f971c9ba2919", "pod": "some-name-rs0-0"}
2025-09-22T15:11:07.013Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "99bfa511-0e8e-485e-a8b6-f971c9ba2919", "pod": "some-name-rs0-2"}
2025-09-22T15:11:07.013Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "99bfa511-0e8e-485e-a8b6-f971c9ba2919", "sts": "some-name-rs0", "replset": "rs0"}
2025-09-22T15:11:07.013Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "99bfa511-0e8e-485e-a8b6-f971c9ba2919", "sts": "some-name-cfg", "replset": "cfg"}
2025-09-22T15:11:07.013Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "99bfa511-0e8e-485e-a8b6-f971c9ba2919", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-22T15:07:01.997935819Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-09-22T15:11:07.197Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "99bfa511-0e8e-485e-a8b6-f971c9ba2919", "status": {"type":"physical","opid":"","name":"2025-09-22T15:07:01.997935819Z","replsets":[{"name":"rs0","start_ts":0,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1758553863,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","error":"wait for nodes in rs: check heartbeat in .pbm.restore/2025-09-22T15:07:01.997935819Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-incremental-sharded-2770.svc.cluster.local:27017.hb: stuck, last beat ts: 1758553622","conditions":null},{"name":"cfg","start_ts":0,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1758553863,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","error":"wait for shards: rs failed: 1758553863:wait for nodes in rs: check heartbeat in .pbm.restore/2025-09-22T15:07:01.997935819Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-incremental-sharded-2770.svc.cluster.local:27017.hb: stuck, last beat ts: 1758553622","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":1758553863,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"error","conditions":null,"n":null,"error":"move to running state: wait for shards: rs failed: 1758553863:wait for nodes in rs: check heartbeat in .pbm.restore/2025-09-22T15:07:01.997935819Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-incremental-sharded-2770.svc.cluster.local:27017.hb: stuck, last beat ts: 1758553622","pbm_version":"","balancer":""}}
2025-09-22T15:11:07.197Z	INFO	Restore state changed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"}, "namespace": "demand-backup-incremental-sharded-2770", "name": "restore-backup-azure-blob-sharded", "reconcileID": "99bfa511-0e8e-485e-a8b6-f971c9ba2919", "previous": "requested", "current": "error"}
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDBRestore
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-2770"},"spec":{"backupName":"backup-azure-blob-sharded","clusterName":"some-name"}}
  creationTimestamp: "2025-09-22T15:03:04Z"
  generation: 1
  name: restore-backup-azure-blob-sharded
  namespace: demand-backup-incremental-sharded-2770
  resourceVersion: "1758553867213951009"
  uid: 0575561f-f7f6-4380-9519-d9d06a9f786f
spec:
  backupName: backup-azure-blob-sharded
  clusterName: some-name
status:
  error: 'move to running state: wait for shards: rs failed: 1758553863:wait for nodes
    in rs: check heartbeat in .pbm.restore/2025-09-22T15:07:01.997935819Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-incremental-sharded-2770.svc.cluster.local:27017.hb:
    stuck, last beat ts: 1758553622'
  pbmName: "2025-09-22T15:07:01.997935819Z"
  state: error
Name:         restore-backup-azure-blob-sharded
Namespace:    demand-backup-incremental-sharded-2770
Labels:       <none>
Annotations:  <none>
API Version:  psmdb.percona.com/v1
Kind:         PerconaServerMongoDBRestore
Metadata:
  Creation Timestamp:  2025-09-22T15:03:04Z
  Generation:          1
  Resource Version:    1758553867213951009
  UID:                 0575561f-f7f6-4380-9519-d9d06a9f786f
Spec:
  Backup Name:   backup-azure-blob-sharded
  Cluster Name:  some-name
Status:
  Error:     move to running state: wait for shards: rs failed: 1758553863:wait for nodes in rs: check heartbeat in .pbm.restore/2025-09-22T15:07:01.997935819Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-incremental-sharded-2770.svc.cluster.local:27017.hb: stuck, last beat ts: 1758553622
  Pbm Name:  2025-09-22T15:07:01.997935819Z
  State:     error
Events:      <none>
Restore object restore-backup-azure-blob-sharded is in error state after 4 minutes.
something went wrong with operator or kubernetes cluster