Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/logs/demand-backup-fs.log
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
+ cluster=some-name
+ create_infra demand-backup-fs-11763
+ local ns=demand-backup-fs-11763
+ echo demand-backup-fs-11763
+ [[ 0 == 1 ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrole
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ awk '{print$1}'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.J8fZ99ZNIh
++ mktemp
+ local LAST_OUT=/tmp/tmp.uWB1awsMb2
+ local LAST_ERR=/tmp/tmp.kebcCB0yZ1
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.H9hBYQBoZu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J8fZ99ZNIh
+ cat /tmp/tmp.kebcCB0yZ1
+ rm /tmp/tmp.J8fZ99ZNIh /tmp/tmp.kebcCB0yZ1
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uWB1awsMb2
+ cat /tmp/tmp.H9hBYQBoZu
+ rm /tmp/tmp.uWB1awsMb2 /tmp/tmp.H9hBYQBoZu
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.52jzsL6PWD
++ mktemp
+ local LAST_ERR=/tmp/tmp.bm807PRoOH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.52jzsL6PWD
+ cat /tmp/tmp.bm807PRoOH
+ rm /tmp/tmp.52jzsL6PWD /tmp/tmp.bm807PRoOH
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.pMzfnkXZOv
++ mktemp
+ local LAST_ERR=/tmp/tmp.FK1l99TkOJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pMzfnkXZOv
namespace/psmdb-operator created
+ cat /tmp/tmp.FK1l99TkOJ
+ rm /tmp/tmp.pMzfnkXZOv /tmp/tmp.FK1l99TkOJ
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H5nR3zwFSw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fx0tM06OyV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.H5nR3zwFSw
++ cat /tmp/tmp.fx0tM06OyV
++ rm /tmp/tmp.H5nR3zwFSw /tmp/tmp.fx0tM06OyV
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster12 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.VQA65YSZRl
++ mktemp
+ local LAST_ERR=/tmp/tmp.JiDW6LE3gP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster12 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VQA65YSZRl
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster12" modified.
+ cat /tmp/tmp.JiDW6LE3gP
+ rm /tmp/tmp.VQA65YSZRl /tmp/tmp.JiDW6LE3gP
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.SG2jOUkQIr
++ mktemp
+ local LAST_ERR=/tmp/tmp.7dunozV5TQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SG2jOUkQIr
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.7dunozV5TQ
+ rm /tmp/tmp.SG2jOUkQIr /tmp/tmp.7dunozV5TQ
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.AxPZJnJKeD
++ mktemp
+ local LAST_ERR=/tmp/tmp.Lluf0uqcOt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AxPZJnJKeD
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.Lluf0uqcOt
+ rm /tmp/tmp.AxPZJnJKeD /tmp/tmp.Lluf0uqcOt
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.HofDQcZuto
++ mktemp
+ local LAST_ERR=/tmp/tmp.MX0yns0Mr4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HofDQcZuto
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.MX0yns0Mr4
+ rm /tmp/tmp.HofDQcZuto /tmp/tmp.MX0yns0Mr4
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dfCe4TRjpj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bSYZCrySS7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dfCe4TRjpj
++ cat /tmp/tmp.bSYZCrySS7
++ rm /tmp/tmp.dfCe4TRjpj /tmp/tmp.bSYZCrySS7
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-859f4bc6d7-vl8jp
+ local pod=percona-server-mongodb-operator-859f4bc6d7-vl8jp
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-859f4bc6d7-vl8jp to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BC5VTaqO4v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r9HoiijDjX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BC5VTaqO4v
++ cat /tmp/tmp.r9HoiijDjX
++ rm /tmp/tmp.BC5VTaqO4v /tmp/tmp.r9HoiijDjX
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-vl8jp
++ mktemp
+ local LAST_OUT=/tmp/tmp.URduSx4FIZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.nin2tgpYTL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-vl8jp
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.URduSx4FIZ
+ cat /tmp/tmp.nin2tgpYTL
+ rm /tmp/tmp.URduSx4FIZ /tmp/tmp.nin2tgpYTL
+ return 0
2026-06-09T07:33:43.180Z	INFO	setup	Manager starting up	{"gitCommit": "fb8cc718776dc8f9e011cd5f23fe48cf7f65143e", "gitBranch": "PR-2058-fb8cc7187", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-fs-11763
+ local namespace=demand-backup-fs-11763
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ awk '{print$1}'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces demand-backup-fs-11763'
+ set +o xtrace
++ mktemp


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-fs-11763
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-fs-11763 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.tJS0aHxsmq
++ mktemp
+ local LAST_OUT=/tmp/tmp.QTHzWkgtDD
+ local LAST_ERR=/tmp/tmp.UxUQ8cw3BQ
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.6fu62RthFj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-fs-11763 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tJS0aHxsmq
+ cat /tmp/tmp.UxUQ8cw3BQ
+ rm /tmp/tmp.tJS0aHxsmq /tmp/tmp.UxUQ8cw3BQ
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QTHzWkgtDD
+ cat /tmp/tmp.6fu62RthFj
+ rm /tmp/tmp.QTHzWkgtDD /tmp/tmp.6fu62RthFj
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-fs-11763
++ mktemp
+ local LAST_OUT=/tmp/tmp.jCY6uHMiEA
++ mktemp
+ local LAST_ERR=/tmp/tmp.EahTCNoO0e
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-fs-11763
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jCY6uHMiEA
+ cat /tmp/tmp.EahTCNoO0e
+ rm /tmp/tmp.jCY6uHMiEA /tmp/tmp.EahTCNoO0e
+ return 0
+ desc 'create namespace demand-backup-fs-11763'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-fs-11763
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-fs-11763
++ mktemp
+ local LAST_OUT=/tmp/tmp.PAH5j67QIA
++ mktemp
+ local LAST_ERR=/tmp/tmp.L0Ry50cKyM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-fs-11763
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PAH5j67QIA
namespace/demand-backup-fs-11763 created
+ cat /tmp/tmp.L0Ry50cKyM
+ rm /tmp/tmp.PAH5j67QIA /tmp/tmp.L0Ry50cKyM
+ return 0
+ set_kube_ctx demand-backup-fs-11763
+ local namespace=demand-backup-fs-11763
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WcpQtpaxDD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TEbVKRjMxJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WcpQtpaxDD
++ cat /tmp/tmp.TEbVKRjMxJ
++ rm /tmp/tmp.WcpQtpaxDD /tmp/tmp.TEbVKRjMxJ
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster12 --namespace=demand-backup-fs-11763
++ mktemp
+ local LAST_OUT=/tmp/tmp.fYlL26klRh
++ mktemp
+ local LAST_ERR=/tmp/tmp.bMqSIfMJbS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster12 --namespace=demand-backup-fs-11763
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fYlL26klRh
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster12" modified.
+ cat /tmp/tmp.bMqSIfMJbS
+ rm /tmp/tmp.fYlL26klRh /tmp/tmp.bMqSIfMJbS
+ return 0
+ kubectl_bin delete ns storage
++ mktemp
+ local LAST_OUT=/tmp/tmp.kGAKAiTyUX
++ mktemp
+ local LAST_ERR=/tmp/tmp.Fz82TD6E2G
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete ns storage
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.kGAKAiTyUX
+ cat /tmp/tmp.Fz82TD6E2G
Error from server (NotFound): namespaces "storage" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete ns storage
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.kGAKAiTyUX
+ cat /tmp/tmp.Fz82TD6E2G
Error from server (NotFound): namespaces "storage" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete ns storage
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.kGAKAiTyUX
+ cat /tmp/tmp.Fz82TD6E2G
Error from server (NotFound): namespaces "storage" not found
+ sleep 8
+ cat /tmp/tmp.kGAKAiTyUX
+ cat /tmp/tmp.Fz82TD6E2G
Error from server (NotFound): namespaces "storage" not found
+ rm /tmp/tmp.kGAKAiTyUX /tmp/tmp.Fz82TD6E2G
+ return 1
+ :
+ [[ 1 != 1 ]]
+ uid=1001
+ [[ -n '' ]]
+ log 'deploying NFS server'
+ set +o xtrace
[2026-06-09T07:34:31+0000] deploying NFS server
+ deploy_nfs_server 1001
+ local uid=1001
+ kubectl_bin create namespace storage
++ mktemp
+ local LAST_OUT=/tmp/tmp.VLSlOVTeq4
++ mktemp
+ local LAST_ERR=/tmp/tmp.1D6UlJqWD6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace storage
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VLSlOVTeq4
namespace/storage created
+ cat /tmp/tmp.1D6UlJqWD6
+ rm /tmp/tmp.VLSlOVTeq4 /tmp/tmp.1D6UlJqWD6
+ return 0
+ kubectl_bin apply -n storage -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/nfs-server.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.tLKg2KXup4
++ mktemp
+ local LAST_ERR=/tmp/tmp.4Q3UtAjOLF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n storage -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/nfs-server.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tLKg2KXup4
serviceaccount/nfs-server created
rolebinding.rbac.authorization.k8s.io/system:openshift:scc:privileged created
persistentvolumeclaim/nfs-pvc created
deployment.apps/nfs-server created
service/nfs-service created
+ cat /tmp/tmp.4Q3UtAjOLF
+ rm /tmp/tmp.tLKg2KXup4 /tmp/tmp.4Q3UtAjOLF
+ return 0
+ sleep 5
++ kubectl_bin get pod -n storage -l app=nfs-server -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ONnDHB7Hay
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H8NcsnhYRu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pod -n storage -l app=nfs-server -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ONnDHB7Hay
++ cat /tmp/tmp.H8NcsnhYRu
++ rm /tmp/tmp.ONnDHB7Hay /tmp/tmp.H8NcsnhYRu
++ return 0
+ local nfsPod=nfs-server-7654469b4d-wr72g
++ kubectl_bin get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k95OVcxdXE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Xz5Gt3NkO5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.k95OVcxdXE
++ cat /tmp/tmp.Xz5Gt3NkO5
++ rm /tmp/tmp.k95OVcxdXE /tmp/tmp.Xz5Gt3NkO5
++ return 0
+ [[ Pending == Running ]]
+ log 'Waiting for nfs-server-7654469b4d-wr72g to start Running'
+ set +o xtrace
[2026-06-09T07:34:42+0000] Waiting for nfs-server-7654469b4d-wr72g to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fihLmErvZZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.h3uOdR6KYA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fihLmErvZZ
++ cat /tmp/tmp.h3uOdR6KYA
++ rm /tmp/tmp.fihLmErvZZ /tmp/tmp.h3uOdR6KYA
++ return 0
+ [[ Pending == Running ]]
+ log 'Waiting for nfs-server-7654469b4d-wr72g to start Running'
+ set +o xtrace
[2026-06-09T07:34:45+0000] Waiting for nfs-server-7654469b4d-wr72g to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OL9D7ljPzf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UyKQNmnhhX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OL9D7ljPzf
++ cat /tmp/tmp.UyKQNmnhhX
++ rm /tmp/tmp.OL9D7ljPzf /tmp/tmp.UyKQNmnhhX
++ return 0
+ [[ Pending == Running ]]
+ log 'Waiting for nfs-server-7654469b4d-wr72g to start Running'
+ set +o xtrace
[2026-06-09T07:34:47+0000] Waiting for nfs-server-7654469b4d-wr72g to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oK3R4EAcOW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HpDI5nGWlR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oK3R4EAcOW
++ cat /tmp/tmp.HpDI5nGWlR
++ rm /tmp/tmp.oK3R4EAcOW /tmp/tmp.HpDI5nGWlR
++ return 0
+ [[ Pending == Running ]]
+ log 'Waiting for nfs-server-7654469b4d-wr72g to start Running'
+ set +o xtrace
[2026-06-09T07:34:49+0000] Waiting for nfs-server-7654469b4d-wr72g to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.x86Yd1aDyR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BNUx0oLbEq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.x86Yd1aDyR
++ cat /tmp/tmp.BNUx0oLbEq
++ rm /tmp/tmp.x86Yd1aDyR /tmp/tmp.BNUx0oLbEq
++ return 0
+ [[ Pending == Running ]]
+ log 'Waiting for nfs-server-7654469b4d-wr72g to start Running'
+ set +o xtrace
[2026-06-09T07:34:51+0000] Waiting for nfs-server-7654469b4d-wr72g to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rzDg8E6vyj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Bef5wD0KDd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rzDg8E6vyj
++ cat /tmp/tmp.Bef5wD0KDd
++ rm /tmp/tmp.rzDg8E6vyj /tmp/tmp.Bef5wD0KDd
++ return 0
+ [[ Pending == Running ]]
+ log 'Waiting for nfs-server-7654469b4d-wr72g to start Running'
+ set +o xtrace
[2026-06-09T07:34:53+0000] Waiting for nfs-server-7654469b4d-wr72g to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PbEJBMAgNx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MRqvQfDmST
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PbEJBMAgNx
++ cat /tmp/tmp.MRqvQfDmST
++ rm /tmp/tmp.PbEJBMAgNx /tmp/tmp.MRqvQfDmST
++ return 0
+ [[ Pending == Running ]]
+ log 'Waiting for nfs-server-7654469b4d-wr72g to start Running'
+ set +o xtrace
[2026-06-09T07:34:55+0000] Waiting for nfs-server-7654469b4d-wr72g to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xCUSZn7Tjo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N8NnojCYb6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pod nfs-server-7654469b4d-wr72g -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xCUSZn7Tjo
++ cat /tmp/tmp.N8NnojCYb6
++ rm /tmp/tmp.xCUSZn7Tjo /tmp/tmp.N8NnojCYb6
++ return 0
+ [[ Running == Running ]]
+ kubectl_bin exec -n storage nfs-server-7654469b4d-wr72g -- mkdir /exports/psmdb-some-name-rs0
++ mktemp
+ local LAST_OUT=/tmp/tmp.ea9PanpBLw
++ mktemp
+ local LAST_ERR=/tmp/tmp.CXlVohmQa9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec -n storage nfs-server-7654469b4d-wr72g -- mkdir /exports/psmdb-some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ea9PanpBLw
+ cat /tmp/tmp.CXlVohmQa9
+ rm /tmp/tmp.ea9PanpBLw /tmp/tmp.CXlVohmQa9
+ return 0
+ kubectl_bin exec -n storage nfs-server-7654469b4d-wr72g -- chown 1001:1001 /exports/psmdb-some-name-rs0
++ mktemp
+ local LAST_OUT=/tmp/tmp.VnCkfwqlv9
++ mktemp
+ local LAST_ERR=/tmp/tmp.6CdhoaEwGR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec -n storage nfs-server-7654469b4d-wr72g -- chown 1001:1001 /exports/psmdb-some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VnCkfwqlv9
+ cat /tmp/tmp.6CdhoaEwGR
+ rm /tmp/tmp.VnCkfwqlv9 /tmp/tmp.6CdhoaEwGR
+ return 0
+ log 'creating secrets and start client'
+ set +o xtrace
[2026-06-09T07:35:00+0000] creating secrets and start client
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.T1HA56e5as
++ mktemp
+ local LAST_ERR=/tmp/tmp.MHLKuYJlaH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.T1HA56e5as
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.MHLKuYJlaH
+ rm /tmp/tmp.T1HA56e5as /tmp/tmp.MHLKuYJlaH
+ return 0
+ [[ -n '' ]]
+ log 'creating PSMDB cluster some-name'
+ set +o xtrace
[2026-06-09T07:35:02+0000] creating PSMDB cluster some-name
+ [[ 1 != 1 ]]
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/some-name.yml
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ /usr/sbin/sed -e s/NAME_SPACE/demand-backup-fs-11763/g
+ local LAST_OUT=/tmp/tmp.SJ2KNBJfSO
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.GLq6H8bGas
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SJ2KNBJfSO
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.GLq6H8bGas
+ rm /tmp/tmp.SJ2KNBJfSO /tmp/tmp.GLq6H8bGas
+ return 0
+ log 'wait for all 3 pods to start'
+ set +o xtrace
[2026-06-09T07:35:05+0000] wait for all 3 pods to start
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready...................OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready................OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b6aZT2c46S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fUhq7vUldu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.b6aZT2c46S
++ cat /tmp/tmp.fUhq7vUldu
++ rm /tmp/tmp.b6aZT2c46S /tmp/tmp.fUhq7vUldu
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.................OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1Iw48vZnoy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rpHQA5cLgp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1Iw48vZnoy
++ cat /tmp/tmp.rpHQA5cLgp
++ rm /tmp/tmp.1Iw48vZnoy /tmp/tmp.rpHQA5cLgp
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YxpUaPWdc2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ds00LxLJPx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YxpUaPWdc2
++ cat /tmp/tmp.ds00LxLJPx
++ rm /tmp/tmp.YxpUaPWdc2 /tmp/tmp.ds00LxLJPx
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness...........
+ [[ 1 == 1 ]]
+ log 'checking if statefulset created with expected config'
+ set +o xtrace
[2026-06-09T07:37:46+0000] checking if statefulset created with expected config
+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.saj8Ql0eBk/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-fs-11763", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.rVgYFemnXa
++ mktemp
+ local LAST_ERR=/tmp/tmp.o5UF6sRGxj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rVgYFemnXa
+ cat /tmp/tmp.o5UF6sRGxj
+ rm /tmp/tmp.rVgYFemnXa /tmp/tmp.o5UF6sRGxj
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.saj8Ql0eBk/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.saj8Ql0eBk/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.saj8Ql0eBk/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/statefulset_some-name-rs0.yml /tmp/tmp.saj8Ql0eBk/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-06-09T07:37:47+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ log 'creating user'
+ set +o xtrace
[2026-06-09T07:37:47+0000] creating user
+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-fs-11763
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wGaMVxnNb9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iAFipKw0ns
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wGaMVxnNb9
++ cat /tmp/tmp.iAFipKw0ns
++ rm /tmp/tmp.wGaMVxnNb9 /tmp/tmp.iAFipKw0ns
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DLI2YRiZe9
++ mktemp
+ local LAST_ERR=/tmp/tmp.uUcFQCCt58
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DLI2YRiZe9
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("125de479-4d29-45e2-ab2b-592e45faa33b") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.uUcFQCCt58
+ rm /tmp/tmp.DLI2YRiZe9 /tmp/tmp.uUcFQCCt58
+ return 0
+ sleep 2
+ log 'write initial data'
+ set +o xtrace
[2026-06-09T07:37:52+0000] write initial data
+ write_data 100500 ''
+ local x=100500
+ local find_prefix=
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UpO3qkkdc5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.M6a79r0Cup
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UpO3qkkdc5
++ cat /tmp/tmp.M6a79r0Cup
++ rm /tmp/tmp.UpO3qkkdc5 /tmp/tmp.M6a79r0Cup
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.BSwDHOrOUm
++ mktemp
+ local LAST_ERR=/tmp/tmp.i0bdw9yrQb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BSwDHOrOUm
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("f3214998-e52e-4e9c-9de2-b989ebe54457") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.i0bdw9yrQb
+ rm /tmp/tmp.BSwDHOrOUm /tmp/tmp.i0bdw9yrQb
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-11763 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-06-09T07:37:54+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-11763 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.femw7m6GIb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kcEeJPbQ7L
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.femw7m6GIb
++ cat /tmp/tmp.kcEeJPbQ7L
++ rm /tmp/tmp.femw7m6GIb /tmp/tmp.kcEeJPbQ7L
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8RmbcvQmsN
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZzppLWEKxd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8RmbcvQmsN
+ cat /tmp/tmp.ZzppLWEKxd
+ rm /tmp/tmp.8RmbcvQmsN /tmp/tmp.ZzppLWEKxd
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/find.json /tmp/tmp.saj8Ql0eBk/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2026-06-09T07:37:11.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2026-06-09T07:37:43.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2026-06-09T07:37:43.000+0000 I listening for the commands

+ desc 'CASE 1: Logical backup and restore'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 1: Logical backup and restore
-----------------------------------------------------------------------------------

+ backup_name=backup-nfs-logical
+ run_backup nfs backup-nfs-logical logical
+ local storage=nfs
+ local backup_name=backup-nfs-logical
+ local type=logical
+ log 'running backup backup-nfs-logical'
+ set +o xtrace
[2026-06-09T07:38:01+0000] running backup backup-nfs-logical
+ yq eval $'.metadata.name = "backup-nfs-logical"\n\t\t\t| .spec.storageName = "nfs"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/backup-nfs.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.XkQOw04Sal
++ mktemp
+ local LAST_ERR=/tmp/tmp.SuCoNJLuoJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XkQOw04Sal
perconaservermongodbbackup.psmdb.percona.com/backup-nfs-logical created
+ cat /tmp/tmp.SuCoNJLuoJ
+ rm /tmp/tmp.XkQOw04Sal /tmp/tmp.SuCoNJLuoJ
+ return 0
+ wait_backup backup-nfs-logical
+ local backup_name=backup-nfs-logical
+ local target_state=ready
+ set +o xtrace
waiting for backup-nfs-logical to reach ready state.......OK
+ run_recovery_check backup-nfs-logical some-name -2nd ''
+ local backup=backup-nfs-logical
+ local cluster=some-name
+ local find_prefix_before=-2nd
+ local find_prefix_after=
+ write_data 100501 -2nd
+ local x=100501
+ local find_prefix=-2nd
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l2uCfS1hKE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DOPyBJaiNr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.l2uCfS1hKE
++ cat /tmp/tmp.DOPyBJaiNr
++ rm /tmp/tmp.l2uCfS1hKE /tmp/tmp.DOPyBJaiNr
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3nxpDCtWY6
++ mktemp
+ local LAST_ERR=/tmp/tmp.2lJeUQ8OMd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3nxpDCtWY6
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c26bb6d5-547b-4422-8c16-be04dd7cebf5") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.2lJeUQ8OMd
+ rm /tmp/tmp.3nxpDCtWY6 /tmp/tmp.2lJeUQ8OMd
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-11763 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-06-09T07:38:19+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-11763 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gvgCAegPqZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DobGuLHWwq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gvgCAegPqZ
++ cat /tmp/tmp.DobGuLHWwq
++ rm /tmp/tmp.gvgCAegPqZ /tmp/tmp.DobGuLHWwq
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9pX2PPf6MT
++ mktemp
+ local LAST_ERR=/tmp/tmp.oIT002VY3p
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9pX2PPf6MT
+ cat /tmp/tmp.oIT002VY3p
+ rm /tmp/tmp.9pX2PPf6MT /tmp/tmp.oIT002VY3p
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/find-2nd.json /tmp/tmp.saj8Ql0eBk/find-2nd
+ run_restore backup-nfs-logical
+ local backup_name=backup-nfs-logical
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/restore.yml
+ log 'running restore restore-backup-nfs-logical'
+ set +o xtrace
[2026-06-09T07:38:22+0000] running restore restore-backup-nfs-logical
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-nfs-logical/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-nfs-logical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.39AejmF3gi
++ mktemp
+ local LAST_ERR=/tmp/tmp.I0IYJ3H5gz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.39AejmF3gi
perconaservermongodbrestore.psmdb.percona.com/restore-backup-nfs-logical created
+ cat /tmp/tmp.I0IYJ3H5gz
+ rm /tmp/tmp.39AejmF3gi /tmp/tmp.I0IYJ3H5gz
+ return 0
+ wait_restore backup-nfs-logical some-name
+ local backup_name=backup-nfs-logical
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-nfs-logical object to be created.OK
Waiting psmdb-restore/restore-backup-nfs-logical to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0hVOeyLQ6t
+++ mktemp
++ local LAST_ERR=/tmp/tmp.akqn9yovCG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0hVOeyLQ6t
++ cat /tmp/tmp.akqn9yovCG
++ rm /tmp/tmp.0hVOeyLQ6t /tmp/tmp.akqn9yovCG
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-11763 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-06-09T07:38:48+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-11763 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D0HUJK0UOf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RaZqJ9pw3f
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D0HUJK0UOf
++ cat /tmp/tmp.RaZqJ9pw3f
++ rm /tmp/tmp.D0HUJK0UOf /tmp/tmp.RaZqJ9pw3f
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WvNBFgnvPY
++ mktemp
+ local LAST_ERR=/tmp/tmp.usPvZyoMz1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WvNBFgnvPY
+ cat /tmp/tmp.usPvZyoMz1
+ rm /tmp/tmp.WvNBFgnvPY /tmp/tmp.usPvZyoMz1
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/find.json /tmp/tmp.saj8Ql0eBk/find
+ desc 'CASE 2: Logical backup and PiTR'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 2: Logical backup and PiTR
-----------------------------------------------------------------------------------

+ backup_name=backup-nfs-logical-pitr
+ run_backup nfs backup-nfs-logical-pitr logical
+ local storage=nfs
+ local backup_name=backup-nfs-logical-pitr
+ local type=logical
+ log 'running backup backup-nfs-logical-pitr'
+ set +o xtrace
[2026-06-09T07:38:51+0000] running backup backup-nfs-logical-pitr
+ yq eval $'.metadata.name = "backup-nfs-logical-pitr"\n\t\t\t| .spec.storageName = "nfs"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/backup-nfs.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.LKukYbcgOM
++ mktemp
+ local LAST_ERR=/tmp/tmp.U5N5YqtkMf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LKukYbcgOM
perconaservermongodbbackup.psmdb.percona.com/backup-nfs-logical-pitr created
+ cat /tmp/tmp.U5N5YqtkMf
+ rm /tmp/tmp.LKukYbcgOM /tmp/tmp.U5N5YqtkMf
+ return 0
+ wait_backup backup-nfs-logical-pitr
+ local backup_name=backup-nfs-logical-pitr
+ local target_state=ready
+ set +o xtrace
waiting for backup-nfs-logical-pitr to reach ready state.......OK
+ write_data 100502 -3rd
+ local x=100502
+ local find_prefix=-3rd
+ run_mongo 'use myApp\n db.test.insert({ x: 100502 })' myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local 'command=use myApp\n db.test.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LnSD9Wfgkc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DKWvVgkFAq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LnSD9Wfgkc
++ cat /tmp/tmp.DKWvVgkFAq
++ rm /tmp/tmp.LnSD9Wfgkc /tmp/tmp.DKWvVgkFAq
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.YZvuChrB2x
++ mktemp
+ local LAST_ERR=/tmp/tmp.KYL7OhzdIw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YZvuChrB2x
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("d5cde857-cf15-49de-956d-d900bd1ab3f4") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.KYL7OhzdIw
+ rm /tmp/tmp.YZvuChrB2x /tmp/tmp.KYL7OhzdIw
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-11763 -3rd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local postfix=-3rd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-06-09T07:39:09+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-11763 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y7pC1WutlJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TmSAgSBtMA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y7pC1WutlJ
++ cat /tmp/tmp.TmSAgSBtMA
++ rm /tmp/tmp.y7pC1WutlJ /tmp/tmp.TmSAgSBtMA
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.nzhMsxBhQe
++ mktemp
+ local LAST_ERR=/tmp/tmp.U0KcuvoBet
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nzhMsxBhQe
+ cat /tmp/tmp.U0KcuvoBet
+ rm /tmp/tmp.nzhMsxBhQe /tmp/tmp.U0KcuvoBet
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/find-3rd.json /tmp/tmp.saj8Ql0eBk/find-3rd
+ run_pitr_check backup-nfs-logical-pitr some-name -3rd
+ local backup=backup-nfs-logical-pitr
+ local cluster=some-name
+ local find_prefix=-3rd
+ wait_for_oplogs some-name
+ local cluster1=some-name
++ jq '.backups.snapshot[0].restoreTo'
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ol62fpW8vy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EfrmHicuON
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ol62fpW8vy
++ cat /tmp/tmp.EfrmHicuON
++ rm /tmp/tmp.ol62fpW8vy /tmp/tmp.EfrmHicuON
++ return 0
+ local backup_last_write=1780990738
+ local retries=0
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.y0aTYH2s73
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.BJGKNSxLYp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.y0aTYH2s73
+++ cat /tmp/tmp.BJGKNSxLYp
+++ rm /tmp/tmp.y0aTYH2s73 /tmp/tmp.BJGKNSxLYp
+++ return 0
++ echo 1780990735
+ local last_chunk=1780990735
+ [[ 1780990735 -gt 1780990738 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.iwubDvEYeu
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.dBN6kukXgE
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.iwubDvEYeu
+++ cat /tmp/tmp.dBN6kukXgE
+++ rm /tmp/tmp.iwubDvEYeu /tmp/tmp.dBN6kukXgE
+++ return 0
++ echo 1780990735
+ last_chunk=1780990735
+ retries=1
++ format_date 1780990735
++ local timestamp=1780990735
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990735 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:38:55
++ format_date 1780990738
++ local timestamp=1780990738
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990738 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:38:58
+ log 'Waiting for last oplog chunk (2026-06-09 07:38:55) to be greater than last write (2026-06-09 07:38:58)'
+ set +o xtrace
[2026-06-09T07:39:17+0000] Waiting for last oplog chunk (2026-06-09 07:38:55) to be greater than last write (2026-06-09 07:38:58)
+ sleep 10
+ [[ 1780990735 -gt 1780990738 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.t5LiOy80IG
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.o5hdCAPLWB
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.t5LiOy80IG
+++ cat /tmp/tmp.o5hdCAPLWB
+++ rm /tmp/tmp.t5LiOy80IG /tmp/tmp.o5hdCAPLWB
+++ return 0
++ echo 1780990738
+ last_chunk=1780990738
+ retries=2
++ format_date 1780990738
++ local timestamp=1780990738
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990738 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:38:58
++ format_date 1780990738
++ local timestamp=1780990738
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990738 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:38:58
+ log 'Waiting for last oplog chunk (2026-06-09 07:38:58) to be greater than last write (2026-06-09 07:38:58)'
+ set +o xtrace
[2026-06-09T07:39:29+0000] Waiting for last oplog chunk (2026-06-09 07:38:58) to be greater than last write (2026-06-09 07:38:58)
+ sleep 10
+ [[ 1780990738 -gt 1780990738 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zTyHALSBuK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.q2oVu9Z4bQ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.zTyHALSBuK
+++ cat /tmp/tmp.q2oVu9Z4bQ
+++ rm /tmp/tmp.zTyHALSBuK /tmp/tmp.q2oVu9Z4bQ
+++ return 0
++ echo 1780990738
+ last_chunk=1780990738
+ retries=3
++ format_date 1780990738
++ local timestamp=1780990738
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990738 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:38:58
++ format_date 1780990738
++ local timestamp=1780990738
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990738 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:38:58
+ log 'Waiting for last oplog chunk (2026-06-09 07:38:58) to be greater than last write (2026-06-09 07:38:58)'
+ set +o xtrace
[2026-06-09T07:39:41+0000] Waiting for last oplog chunk (2026-06-09 07:38:58) to be greater than last write (2026-06-09 07:38:58)
+ sleep 10
+ [[ 1780990738 -gt 1780990738 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tZ8IOoyBwK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9anomtUzk5
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tZ8IOoyBwK
+++ cat /tmp/tmp.9anomtUzk5
+++ rm /tmp/tmp.tZ8IOoyBwK /tmp/tmp.9anomtUzk5
+++ return 0
++ echo 1780990738
+ last_chunk=1780990738
+ retries=4
++ format_date 1780990738
++ local timestamp=1780990738
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990738 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:38:58
++ format_date 1780990738
++ local timestamp=1780990738
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990738 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:38:58
+ log 'Waiting for last oplog chunk (2026-06-09 07:38:58) to be greater than last write (2026-06-09 07:38:58)'
+ set +o xtrace
[2026-06-09T07:39:53+0000] Waiting for last oplog chunk (2026-06-09 07:38:58) to be greater than last write (2026-06-09 07:38:58)
+ sleep 10
+ [[ 1780990738 -gt 1780990738 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ local LAST_OUT=/tmp/tmp.Q1mNq321Wb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iyviRhsO6j
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Q1mNq321Wb
+++ cat /tmp/tmp.iyviRhsO6j
+++ rm /tmp/tmp.Q1mNq321Wb /tmp/tmp.iyviRhsO6j
+++ return 0
++ echo 1780990792
+ last_chunk=1780990792
+ retries=5
++ format_date 1780990792
++ local timestamp=1780990792
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990792 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:39:52
++ format_date 1780990738
++ local timestamp=1780990738
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990738 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:38:58
+ log 'Waiting for last oplog chunk (2026-06-09 07:39:52) to be greater than last write (2026-06-09 07:38:58)'
+ set +o xtrace
[2026-06-09T07:40:05+0000] Waiting for last oplog chunk (2026-06-09 07:39:52) to be greater than last write (2026-06-09 07:38:58)
+ sleep 10
+ [[ 1780990792 -gt 1780990738 ]]
+++ get_latest_oplog_chunk_ts some-name
+++ local cluster=some-name
++++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.pP7nssLbZo
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.x4iNP3xJPH
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.pP7nssLbZo
++++ cat /tmp/tmp.x4iNP3xJPH
++++ rm /tmp/tmp.pP7nssLbZo /tmp/tmp.x4iNP3xJPH
++++ return 0
+++ echo 1780990792
++ format_date 1780990792
++ local timestamp=1780990792
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990792 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:39:52
+ local 'target_time=2026-06-09 07:39:52'
+ log 'dropping test collection'
+ set +o xtrace
[2026-06-09T07:40:17+0000] dropping test collection
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OFFQGuJMaU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5hF7qsEGSl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OFFQGuJMaU
++ cat /tmp/tmp.5hF7qsEGSl
++ rm /tmp/tmp.OFFQGuJMaU /tmp/tmp.5hF7qsEGSl
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZRO3fASwk7
++ mktemp
+ local LAST_ERR=/tmp/tmp.XQ6bhqSSl8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZRO3fASwk7
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b38a8156-ae97-454f-b6ff-0606c09840f9") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.XQ6bhqSSl8
+ rm /tmp/tmp.ZRO3fASwk7 /tmp/tmp.XQ6bhqSSl8
+ return 0
+ log 'checking pitr... backup: backup-nfs-logical-pitr target: 2026-06-09 07:39:52'
+ set +o xtrace
[2026-06-09T07:40:19+0000] checking pitr... backup: backup-nfs-logical-pitr target: 2026-06-09 07:39:52
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/pitr.yml
+ yq eval '.metadata.name = "restore-backup-nfs-logical-pitr"'
+ yq eval '.spec.backupName = "backup-nfs-logical-pitr"'
+ yq eval '.spec.pitr.date = "2026-06-09 07:39:52"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.4wwVNlOCmZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.E0W6qDNAn9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4wwVNlOCmZ
perconaservermongodbrestore.psmdb.percona.com/restore-backup-nfs-logical-pitr created
+ cat /tmp/tmp.E0W6qDNAn9
+ rm /tmp/tmp.4wwVNlOCmZ /tmp/tmp.E0W6qDNAn9
+ return 0
+ wait_restore backup-nfs-logical-pitr some-name
+ local backup_name=backup-nfs-logical-pitr
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-nfs-logical-pitr object to be created.OK
Waiting psmdb-restore/restore-backup-nfs-logical-pitr to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Naz4ecpLS9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.baYTCCN7TU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Naz4ecpLS9
++ cat /tmp/tmp.baYTCCN7TU
++ rm /tmp/tmp.Naz4ecpLS9 /tmp/tmp.baYTCCN7TU
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-11763 -3rd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local postfix=-3rd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-06-09T07:41:02+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-11763 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cWyKcQlfyC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o8F0BDdGQd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cWyKcQlfyC
++ cat /tmp/tmp.o8F0BDdGQd
++ rm /tmp/tmp.cWyKcQlfyC /tmp/tmp.o8F0BDdGQd
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.p2ZRVCU6va
++ mktemp
+ local LAST_ERR=/tmp/tmp.EvSWUVh6Hl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.p2ZRVCU6va
+ cat /tmp/tmp.EvSWUVh6Hl
+ rm /tmp/tmp.p2ZRVCU6va /tmp/tmp.EvSWUVh6Hl
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/find-3rd.json /tmp/tmp.saj8Ql0eBk/find-3rd
+ desc 'CASE 3: Physical backup and restore'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 3: Physical backup and restore
-----------------------------------------------------------------------------------

+ backup_name=backup-nfs-physical
+ run_backup nfs backup-nfs-physical physical
+ local storage=nfs
+ local backup_name=backup-nfs-physical
+ local type=physical
+ log 'running backup backup-nfs-physical'
+ set +o xtrace
[2026-06-09T07:41:05+0000] running backup backup-nfs-physical
+ yq eval $'.metadata.name = "backup-nfs-physical"\n\t\t\t| .spec.storageName = "nfs"\n\t\t\t| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/backup-nfs.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.EVRkmpguXW
++ mktemp
+ local LAST_ERR=/tmp/tmp.i1QIJW3sVq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EVRkmpguXW
perconaservermongodbbackup.psmdb.percona.com/backup-nfs-physical created
+ cat /tmp/tmp.i1QIJW3sVq
+ rm /tmp/tmp.EVRkmpguXW /tmp/tmp.i1QIJW3sVq
+ return 0
+ wait_backup backup-nfs-physical
+ local backup_name=backup-nfs-physical
+ local target_state=ready
+ set +o xtrace
waiting for backup-nfs-physical to reach ready state.......OK
+ run_recovery_check backup-nfs-physical some-name -4th -3rd
+ local backup=backup-nfs-physical
+ local cluster=some-name
+ local find_prefix_before=-4th
+ local find_prefix_after=-3rd
+ write_data 100501 -4th
+ local x=100501
+ local find_prefix=-4th
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OHWAqVYiQC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UHVW5d3RxB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OHWAqVYiQC
++ cat /tmp/tmp.UHVW5d3RxB
++ rm /tmp/tmp.OHWAqVYiQC /tmp/tmp.UHVW5d3RxB
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.r7xb8drJvM
++ mktemp
+ local LAST_ERR=/tmp/tmp.tVnKnKvXj0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.r7xb8drJvM
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("4742e6d1-d718-47f4-a674-de641e8e29a7") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.tVnKnKvXj0
+ rm /tmp/tmp.r7xb8drJvM /tmp/tmp.tVnKnKvXj0
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-11763 -4th .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local postfix=-4th
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-06-09T07:41:21+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-11763 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.62Y750VXLw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fIZWsjV3xD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.62Y750VXLw
++ cat /tmp/tmp.fIZWsjV3xD
++ rm /tmp/tmp.62Y750VXLw /tmp/tmp.fIZWsjV3xD
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.MAnr3px5dF
++ mktemp
+ local LAST_ERR=/tmp/tmp.5wWm15ffmR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MAnr3px5dF
+ cat /tmp/tmp.5wWm15ffmR
+ rm /tmp/tmp.MAnr3px5dF /tmp/tmp.5wWm15ffmR
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/find-4th.json /tmp/tmp.saj8Ql0eBk/find-4th
+ run_restore backup-nfs-physical
+ local backup_name=backup-nfs-physical
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/restore.yml
+ log 'running restore restore-backup-nfs-physical'
+ set +o xtrace
[2026-06-09T07:41:24+0000] running restore restore-backup-nfs-physical
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-nfs-physical/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-nfs-physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZT8jDO1G1B
++ mktemp
+ local LAST_ERR=/tmp/tmp.Cr8mE2q88c
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZT8jDO1G1B
perconaservermongodbrestore.psmdb.percona.com/restore-backup-nfs-physical created
+ cat /tmp/tmp.Cr8mE2q88c
+ rm /tmp/tmp.ZT8jDO1G1B /tmp/tmp.Cr8mE2q88c
+ return 0
+ wait_restore backup-nfs-physical some-name
+ local backup_name=backup-nfs-physical
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-nfs-physical object to be created.OK
Waiting psmdb-restore/restore-backup-nfs-physical to reach state "ready" .....OK after 4 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hCizq5nuvK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mSdlWv22Np
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hCizq5nuvK
++ cat /tmp/tmp.mSdlWv22Np
++ rm /tmp/tmp.hCizq5nuvK /tmp/tmp.mSdlWv22Np
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.r3F6QQFI26
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ve8uXXgTcj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.r3F6QQFI26
++ cat /tmp/tmp.Ve8uXXgTcj
++ rm /tmp/tmp.r3F6QQFI26 /tmp/tmp.Ve8uXXgTcj
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UfaMnU1Nm2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q1Q0HUFp2X
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UfaMnU1Nm2
++ cat /tmp/tmp.q1Q0HUFp2X
++ rm /tmp/tmp.UfaMnU1Nm2 /tmp/tmp.q1Q0HUFp2X
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uXSGPxo55U
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pgDFArqG5U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uXSGPxo55U
++ cat /tmp/tmp.pgDFArqG5U
++ rm /tmp/tmp.uXSGPxo55U /tmp/tmp.pgDFArqG5U
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AuwlRGI7EY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Jzz1dEb4XH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AuwlRGI7EY
++ cat /tmp/tmp.Jzz1dEb4XH
++ rm /tmp/tmp.AuwlRGI7EY /tmp/tmp.Jzz1dEb4XH
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pPFhmDsbD6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gtZ9Jw6cAc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pPFhmDsbD6
++ cat /tmp/tmp.gtZ9Jw6cAc
++ rm /tmp/tmp.pPFhmDsbD6 /tmp/tmp.gtZ9Jw6cAc
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X3uQHMXwi6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1s3oFsVb85
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.X3uQHMXwi6
++ cat /tmp/tmp.1s3oFsVb85
++ rm /tmp/tmp.X3uQHMXwi6 /tmp/tmp.1s3oFsVb85
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MGOXbLHTqn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y7jFyQnCQJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MGOXbLHTqn
++ cat /tmp/tmp.y7jFyQnCQJ
++ rm /tmp/tmp.MGOXbLHTqn /tmp/tmp.y7jFyQnCQJ
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2IzQoHAj2z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1v6dneQfgZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2IzQoHAj2z
++ cat /tmp/tmp.1v6dneQfgZ
++ rm /tmp/tmp.2IzQoHAj2z /tmp/tmp.1v6dneQfgZ
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-11763 -3rd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local postfix=-3rd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-06-09T07:47:20+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-11763 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6tjmXIEzla
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q2Ks5XfOfq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6tjmXIEzla
++ cat /tmp/tmp.q2Ks5XfOfq
++ rm /tmp/tmp.6tjmXIEzla /tmp/tmp.q2Ks5XfOfq
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Gwm8lF7oEG
++ mktemp
+ local LAST_ERR=/tmp/tmp.yJFv5sA6Ba
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Gwm8lF7oEG
+ cat /tmp/tmp.yJFv5sA6Ba
+ rm /tmp/tmp.Gwm8lF7oEG /tmp/tmp.yJFv5sA6Ba
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/find-3rd.json /tmp/tmp.saj8Ql0eBk/find-3rd
+ desc 'CASE 4: Physical backup and PiTR'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 4: Physical backup and PiTR
-----------------------------------------------------------------------------------

+ backup_name=backup-nfs-physical-pitr
+ run_backup nfs backup-nfs-physical-pitr physical
+ local storage=nfs
+ local backup_name=backup-nfs-physical-pitr
+ local type=physical
+ log 'running backup backup-nfs-physical-pitr'
+ set +o xtrace
[2026-06-09T07:47:22+0000] running backup backup-nfs-physical-pitr
+ yq eval $'.metadata.name = "backup-nfs-physical-pitr"\n\t\t\t| .spec.storageName = "nfs"\n\t\t\t| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/backup-nfs.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.J2bAKbS4Qd
++ mktemp
+ local LAST_ERR=/tmp/tmp.w1GVsrNfq6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J2bAKbS4Qd
perconaservermongodbbackup.psmdb.percona.com/backup-nfs-physical-pitr created
+ cat /tmp/tmp.w1GVsrNfq6
+ rm /tmp/tmp.J2bAKbS4Qd /tmp/tmp.w1GVsrNfq6
+ return 0
+ wait_backup backup-nfs-physical-pitr
+ local backup_name=backup-nfs-physical-pitr
+ local target_state=ready
+ set +o xtrace
waiting for backup-nfs-physical-pitr to reach ready state.......OK
+ write_data 100503 -5th
+ local x=100503
+ local find_prefix=-5th
+ run_mongo 'use myApp\n db.test.insert({ x: 100503 })' myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local 'command=use myApp\n db.test.insert({ x: 100503 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PnJXvqwu3P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Mbi0Xen2HJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PnJXvqwu3P
++ cat /tmp/tmp.Mbi0Xen2HJ
++ rm /tmp/tmp.PnJXvqwu3P /tmp/tmp.Mbi0Xen2HJ
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rtKRf0I7mh
++ mktemp
+ local LAST_ERR=/tmp/tmp.J0xJgeZQzJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rtKRf0I7mh
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8fabf9e5-499a-4404-abf3-a919a121b2f9") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.J0xJgeZQzJ
+ rm /tmp/tmp.rtKRf0I7mh /tmp/tmp.J0xJgeZQzJ
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-11763 -5th .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local postfix=-5th
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-06-09T07:47:39+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-11763 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EATr4S0Tnv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CTZozcuAUa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EATr4S0Tnv
++ cat /tmp/tmp.CTZozcuAUa
++ rm /tmp/tmp.EATr4S0Tnv /tmp/tmp.CTZozcuAUa
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vYDUruhIjK
++ mktemp
+ local LAST_ERR=/tmp/tmp.ldfUXl5Fe7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vYDUruhIjK
+ cat /tmp/tmp.ldfUXl5Fe7
+ rm /tmp/tmp.vYDUruhIjK /tmp/tmp.ldfUXl5Fe7
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/find-5th.json /tmp/tmp.saj8Ql0eBk/find-5th
+ run_pitr_check backup-nfs-physical-pitr some-name -5th
+ local backup=backup-nfs-physical-pitr
+ local cluster=some-name
+ local find_prefix=-5th
+ wait_for_oplogs some-name
+ local cluster1=some-name
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ mktemp
++ jq '.backups.snapshot[0].restoreTo'
++ local LAST_OUT=/tmp/tmp.ntHDJ3JeO6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rLiUG2KVMr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ntHDJ3JeO6
++ cat /tmp/tmp.rLiUG2KVMr
++ rm /tmp/tmp.ntHDJ3JeO6 /tmp/tmp.rLiUG2KVMr
++ return 0
+ local backup_last_write=1780991247
+ local retries=0
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zyffqNbJHM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5mlYMkxnh7
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.zyffqNbJHM
+++ cat /tmp/tmp.5mlYMkxnh7
+++ rm /tmp/tmp.zyffqNbJHM /tmp/tmp.5mlYMkxnh7
+++ return 0
++ echo 1780990840
+ local last_chunk=1780990840
+ [[ 1780990840 -gt 1780991247 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.KtaptF46ii
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ScPkAnLtwb
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.KtaptF46ii
+++ cat /tmp/tmp.ScPkAnLtwb
+++ rm /tmp/tmp.KtaptF46ii /tmp/tmp.ScPkAnLtwb
+++ return 0
++ echo 1780990840
+ last_chunk=1780990840
+ retries=1
++ format_date 1780990840
++ local timestamp=1780990840
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990840 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:40:40
++ format_date 1780991247
++ local timestamp=1780991247
+++ TZ=UTC
+++ /usr/sbin/date -d@1780991247 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:47:27
+ log 'Waiting for last oplog chunk (2026-06-09 07:40:40) to be greater than last write (2026-06-09 07:47:27)'
+ set +o xtrace
[2026-06-09T07:47:47+0000] Waiting for last oplog chunk (2026-06-09 07:40:40) to be greater than last write (2026-06-09 07:47:27)
+ sleep 10
+ [[ 1780990840 -gt 1780991247 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Q1B7u3ODJB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.k9gIMQXs1o
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Q1B7u3ODJB
+++ cat /tmp/tmp.k9gIMQXs1o
+++ rm /tmp/tmp.Q1B7u3ODJB /tmp/tmp.k9gIMQXs1o
+++ return 0
++ echo 1780990840
+ last_chunk=1780990840
+ retries=2
++ format_date 1780990840
++ local timestamp=1780990840
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990840 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:40:40
++ format_date 1780991247
++ local timestamp=1780991247
+++ TZ=UTC
+++ /usr/sbin/date -d@1780991247 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:47:27
+ log 'Waiting for last oplog chunk (2026-06-09 07:40:40) to be greater than last write (2026-06-09 07:47:27)'
+ set +o xtrace
[2026-06-09T07:47:59+0000] Waiting for last oplog chunk (2026-06-09 07:40:40) to be greater than last write (2026-06-09 07:47:27)
+ sleep 10
+ [[ 1780990840 -gt 1780991247 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.pjKPtFq5LK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.dgH3MpQD72
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.pjKPtFq5LK
+++ cat /tmp/tmp.dgH3MpQD72
+++ rm /tmp/tmp.pjKPtFq5LK /tmp/tmp.dgH3MpQD72
+++ return 0
++ echo 1780990840
+ last_chunk=1780990840
+ retries=3
++ format_date 1780990840
++ local timestamp=1780990840
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990840 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:40:40
++ format_date 1780991247
++ local timestamp=1780991247
+++ TZ=UTC
+++ /usr/sbin/date -d@1780991247 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:47:27
+ log 'Waiting for last oplog chunk (2026-06-09 07:40:40) to be greater than last write (2026-06-09 07:47:27)'
+ set +o xtrace
[2026-06-09T07:48:11+0000] Waiting for last oplog chunk (2026-06-09 07:40:40) to be greater than last write (2026-06-09 07:47:27)
+ sleep 10
+ [[ 1780990840 -gt 1780991247 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2PGfnDOFMg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.BRUWDNJf7x
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.2PGfnDOFMg
+++ cat /tmp/tmp.BRUWDNJf7x
+++ rm /tmp/tmp.2PGfnDOFMg /tmp/tmp.BRUWDNJf7x
+++ return 0
++ echo 1780990840
+ last_chunk=1780990840
+ retries=4
++ format_date 1780990840
++ local timestamp=1780990840
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990840 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:40:40
++ format_date 1780991247
++ local timestamp=1780991247
+++ TZ=UTC
+++ /usr/sbin/date -d@1780991247 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:47:27
+ log 'Waiting for last oplog chunk (2026-06-09 07:40:40) to be greater than last write (2026-06-09 07:47:27)'
+ set +o xtrace
[2026-06-09T07:48:23+0000] Waiting for last oplog chunk (2026-06-09 07:40:40) to be greater than last write (2026-06-09 07:47:27)
+ sleep 10
+ [[ 1780990840 -gt 1780991247 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.riVBxCvvIu
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.tD1nQLTyLd
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.riVBxCvvIu
+++ cat /tmp/tmp.tD1nQLTyLd
+++ rm /tmp/tmp.riVBxCvvIu /tmp/tmp.tD1nQLTyLd
+++ return 0
++ echo 1780990840
+ last_chunk=1780990840
+ retries=5
++ format_date 1780990840
++ local timestamp=1780990840
+++ TZ=UTC
+++ /usr/sbin/date -d@1780990840 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:40:40
++ format_date 1780991247
++ local timestamp=1780991247
+++ TZ=UTC
+++ /usr/sbin/date -d@1780991247 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:47:27
+ log 'Waiting for last oplog chunk (2026-06-09 07:40:40) to be greater than last write (2026-06-09 07:47:27)'
+ set +o xtrace
[2026-06-09T07:48:34+0000] Waiting for last oplog chunk (2026-06-09 07:40:40) to be greater than last write (2026-06-09 07:47:27)
+ sleep 10
+ [[ 1780990840 -gt 1780991247 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.7NgGs40XmS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.l8kPyLfHjc
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.7NgGs40XmS
+++ cat /tmp/tmp.l8kPyLfHjc
+++ rm /tmp/tmp.7NgGs40XmS /tmp/tmp.l8kPyLfHjc
+++ return 0
++ echo 1780991325
+ last_chunk=1780991325
+ retries=6
++ format_date 1780991325
++ local timestamp=1780991325
+++ TZ=UTC
+++ /usr/sbin/date -d@1780991325 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:48:45
++ format_date 1780991247
++ local timestamp=1780991247
+++ TZ=UTC
+++ /usr/sbin/date -d@1780991247 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:47:27
+ log 'Waiting for last oplog chunk (2026-06-09 07:48:45) to be greater than last write (2026-06-09 07:47:27)'
+ set +o xtrace
[2026-06-09T07:48:46+0000] Waiting for last oplog chunk (2026-06-09 07:48:45) to be greater than last write (2026-06-09 07:47:27)
+ sleep 10
+ [[ 1780991325 -gt 1780991247 ]]
+++ get_latest_oplog_chunk_ts some-name
+++ local cluster=some-name
++++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.Cdrk0eibHE
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.gW0wtxSkKd
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in $(seq 0 2)
++++ set +e
++++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.Cdrk0eibHE
++++ cat /tmp/tmp.gW0wtxSkKd
++++ rm /tmp/tmp.Cdrk0eibHE /tmp/tmp.gW0wtxSkKd
++++ return 0
+++ echo 1780991325
++ format_date 1780991325
++ local timestamp=1780991325
+++ TZ=UTC
+++ /usr/sbin/date -d@1780991325 '+%Y-%m-%d %H:%M:%S'
++ echo 2026-06-09 07:48:45
+ local 'target_time=2026-06-09 07:48:45'
+ log 'dropping test collection'
+ set +o xtrace
[2026-06-09T07:48:58+0000] dropping test collection
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QKVHfagKWl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HfXiRNwln8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QKVHfagKWl
++ cat /tmp/tmp.HfXiRNwln8
++ rm /tmp/tmp.QKVHfagKWl /tmp/tmp.HfXiRNwln8
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.PakhkCXrCf
++ mktemp
+ local LAST_ERR=/tmp/tmp.CHc9MxlJbV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PakhkCXrCf
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-11763.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c0ad810b-1601-45d3-b7d5-fa0ff0415dd6") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.CHc9MxlJbV
+ rm /tmp/tmp.PakhkCXrCf /tmp/tmp.CHc9MxlJbV
+ return 0
+ log 'checking pitr... backup: backup-nfs-physical-pitr target: 2026-06-09 07:48:45'
+ set +o xtrace
[2026-06-09T07:49:01+0000] checking pitr... backup: backup-nfs-physical-pitr target: 2026-06-09 07:48:45
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/conf/pitr.yml
+ yq eval '.metadata.name = "restore-backup-nfs-physical-pitr"'
+ yq eval '.spec.backupName = "backup-nfs-physical-pitr"'
+ yq eval '.spec.pitr.date = "2026-06-09 07:48:45"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ND6ZOxignI
++ mktemp
+ local LAST_ERR=/tmp/tmp.w2YBmKz7Ij
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ND6ZOxignI
perconaservermongodbrestore.psmdb.percona.com/restore-backup-nfs-physical-pitr created
+ cat /tmp/tmp.w2YBmKz7Ij
+ rm /tmp/tmp.ND6ZOxignI /tmp/tmp.w2YBmKz7Ij
+ return 0
+ wait_restore backup-nfs-physical-pitr some-name
+ local backup_name=backup-nfs-physical-pitr
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-nfs-physical-pitr object to be created.OK
Waiting psmdb-restore/restore-backup-nfs-physical-pitr to reach state "ready" .....OK after 4 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HfyLVSZ8xc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5rkKdwtxhJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HfyLVSZ8xc
++ cat /tmp/tmp.5rkKdwtxhJ
++ rm /tmp/tmp.HfyLVSZ8xc /tmp/tmp.5rkKdwtxhJ
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.76saqZRZVh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gUG3tuSuEW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.76saqZRZVh
++ cat /tmp/tmp.gUG3tuSuEW
++ rm /tmp/tmp.76saqZRZVh /tmp/tmp.gUG3tuSuEW
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CDyaQHBZ2G
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fCwkf2qr1z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CDyaQHBZ2G
++ cat /tmp/tmp.fCwkf2qr1z
++ rm /tmp/tmp.CDyaQHBZ2G /tmp/tmp.fCwkf2qr1z
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C7SPs9KNUK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cheNl7MOxR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C7SPs9KNUK
++ cat /tmp/tmp.cheNl7MOxR
++ rm /tmp/tmp.C7SPs9KNUK /tmp/tmp.cheNl7MOxR
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rrlP5Xxsu9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HzUQLnJ5sk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rrlP5Xxsu9
++ cat /tmp/tmp.HzUQLnJ5sk
++ rm /tmp/tmp.rrlP5Xxsu9 /tmp/tmp.HzUQLnJ5sk
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N3kBdkUb4K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.w94GDNXfct
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.N3kBdkUb4K
++ cat /tmp/tmp.w94GDNXfct
++ rm /tmp/tmp.N3kBdkUb4K /tmp/tmp.w94GDNXfct
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p5gkNOJQrx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QiaLGUIeeU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.p5gkNOJQrx
++ cat /tmp/tmp.QiaLGUIeeU
++ rm /tmp/tmp.p5gkNOJQrx /tmp/tmp.QiaLGUIeeU
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xtDB2sEkrS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pozIpE34l0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xtDB2sEkrS
++ cat /tmp/tmp.pozIpE34l0
++ rm /tmp/tmp.xtDB2sEkrS /tmp/tmp.pozIpE34l0
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-11763 -5th .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local postfix=-5th
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-06-09T07:54:56+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-11763 mongodb .svc.cluster.local ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-11763
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-11763 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zcDXObpzEu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ekzlrFXZIy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zcDXObpzEu
++ cat /tmp/tmp.ekzlrFXZIy
++ rm /tmp/tmp.zcDXObpzEu /tmp/tmp.ekzlrFXZIy
++ return 0
+ local client_container=psmdb-client-bb8b97679-nmfwr
+ kubectl_bin exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ny8ZYYipkP
++ mktemp
+ local LAST_ERR=/tmp/tmp.0AUOmRWhYD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-nmfwr -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-11763.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ny8ZYYipkP
+ cat /tmp/tmp.0AUOmRWhYD
+ rm /tmp/tmp.ny8ZYYipkP /tmp/tmp.0AUOmRWhYD
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-fs/compare/find-5th.json /tmp/tmp.saj8Ql0eBk/find-5th
+ destroy demand-backup-fs-11763
+ local namespace=demand-backup-fs-11763
+ local ignore_logs=true
+ [[ 1 == 1 ]]
+ echo 'SKIP_DELETE=1, not destroying demand-backup-fs-11763'
SKIP_DELETE=1, not destroying demand-backup-fs-11763
+ return
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------