Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/logs/demand-backup-fs.log
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
+ cluster=some-name
+ create_infra demand-backup-fs-25161
+ local ns=demand-backup-fs-25161
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.UrabXXBj4I
++ mktemp
+ local LAST_ERR=/tmp/tmp.QdeL1dPmoH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UrabXXBj4I
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.QdeL1dPmoH
+ rm /tmp/tmp.UrabXXBj4I /tmp/tmp.QdeL1dPmoH
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-fs-14722 backup-nfs-logical --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-nfs-logical patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-fs-14722 backup-nfs-logical-pitr --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-nfs-logical-pitr patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-fs-14722 backup-nfs-physical --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-nfs-physical patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.N1MlFzujtC
++ mktemp
+ local LAST_ERR=/tmp/tmp.yNzPaptIR1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N1MlFzujtC
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.yNzPaptIR1
+ rm /tmp/tmp.N1MlFzujtC /tmp/tmp.yNzPaptIR1
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.z8Ca0T3U68
++ mktemp
+ local LAST_ERR=/tmp/tmp.Qlt6cEss7b
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.z8Ca0T3U68
+ cat /tmp/tmp.Qlt6cEss7b
+ rm /tmp/tmp.z8Ca0T3U68 /tmp/tmp.Qlt6cEss7b
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n demand-backup-fs-14722 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.mkXZ5sJaS0
++ mktemp
+ local LAST_ERR=/tmp/tmp.pw1wPAyYmJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mkXZ5sJaS0
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met
+ cat /tmp/tmp.pw1wPAyYmJ
+ rm /tmp/tmp.mkXZ5sJaS0 /tmp/tmp.pw1wPAyYmJ
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.75NOBgqLBA
++ mktemp
+ local LAST_ERR=/tmp/tmp.qJSYxjleoI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.75NOBgqLBA
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.qJSYxjleoI
+ rm /tmp/tmp.75NOBgqLBA /tmp/tmp.qJSYxjleoI
+ return 0
+ check_crd_for_deletion PR-1969-c5cc2da4
+ local git_tag=PR-1969-c5cc2da4
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1969-c5cc2da4/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
++ /usr/bin/sed s/---//g
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2papwH3OY7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dee2VpmbxZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.2papwH3OY7
++ cat /tmp/tmp.dee2VpmbxZ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.2papwH3OY7
++ cat /tmp/tmp.dee2VpmbxZ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.2papwH3OY7
++ cat /tmp/tmp.dee2VpmbxZ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.2papwH3OY7
++ cat /tmp/tmp.dee2VpmbxZ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.2papwH3OY7 /tmp/tmp.dee2VpmbxZ
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
++ mktemp
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ local LAST_OUT=/tmp/tmp.t2U5JWNrRt
+ awk '{print$1}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.L3kZAj1kyR
+ local LAST_ERR=/tmp/tmp.LmMXo6eA8b
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.GijloK08X5
+ local exit_status=0
+ local timeout=4
+ for i in '$(seq 0 2)'
++ seq 0 2
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t2U5JWNrRt
+ cat /tmp/tmp.LmMXo6eA8b
+ rm /tmp/tmp.t2U5JWNrRt /tmp/tmp.LmMXo6eA8b
+ return 0
namespace "demand-backup-fs-14722" deleted
namespace "gke-managed-cim" deleted
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
namespace "storage" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.L3kZAj1kyR
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.GijloK08X5
+ rm /tmp/tmp.L3kZAj1kyR /tmp/tmp.GijloK08X5
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.QpGIigYDIk
++ mktemp
+ local LAST_ERR=/tmp/tmp.jJfcnOBbq4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QpGIigYDIk
+ cat /tmp/tmp.jJfcnOBbq4
+ rm /tmp/tmp.QpGIigYDIk /tmp/tmp.jJfcnOBbq4
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.X4coYmmJqO
++ mktemp
+ local LAST_ERR=/tmp/tmp.xgvMPCnRB2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.X4coYmmJqO
namespace/psmdb-operator created
+ cat /tmp/tmp.xgvMPCnRB2
+ rm /tmp/tmp.X4coYmmJqO /tmp/tmp.xgvMPCnRB2
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SG9GK0jRNW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zKwG5mt0qi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SG9GK0jRNW
++ cat /tmp/tmp.zKwG5mt0qi
++ rm /tmp/tmp.SG9GK0jRNW /tmp/tmp.zKwG5mt0qi
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1969-c5cc2da4-1-cluster4 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.KvMd7JtjEh
++ mktemp
+ local LAST_ERR=/tmp/tmp.KADLjBLXHb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1969-c5cc2da4-1-cluster4 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KvMd7JtjEh
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1969-c5cc2da4-1-cluster4" modified.
+ cat /tmp/tmp.KADLjBLXHb
+ rm /tmp/tmp.KvMd7JtjEh /tmp/tmp.KADLjBLXHb
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.qE85TpVWDn
++ mktemp
+ local LAST_ERR=/tmp/tmp.AbtYEuu66A
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qE85TpVWDn
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.AbtYEuu66A
+ rm /tmp/tmp.qE85TpVWDn /tmp/tmp.AbtYEuu66A
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.tGBMAkgERp
++ mktemp
+ local LAST_ERR=/tmp/tmp.QzYjHGzLpG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tGBMAkgERp
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.QzYjHGzLpG
+ rm /tmp/tmp.tGBMAkgERp /tmp/tmp.QzYjHGzLpG
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1969-c5cc2da4") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.JIWUujhM5g
++ mktemp
+ local LAST_ERR=/tmp/tmp.6eTZ2mWRY1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JIWUujhM5g
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.6eTZ2mWRY1
+ rm /tmp/tmp.JIWUujhM5g /tmp/tmp.6eTZ2mWRY1
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cz2pLnB5tX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TWE5mMJ3zU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cz2pLnB5tX
++ cat /tmp/tmp.TWE5mMJ3zU
++ rm /tmp/tmp.cz2pLnB5tX /tmp/tmp.TWE5mMJ3zU
++ return 0
+ wait_pod percona-server-mongodb-operator-6fc6c78cc6-6ljx9
+ local pod=percona-server-mongodb-operator-6fc6c78cc6-6ljx9
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-6fc6c78cc6-6ljx9 to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QmUqfwqQo0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GdnCpQ7Vka
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QmUqfwqQo0
++ cat /tmp/tmp.GdnCpQ7Vka
++ rm /tmp/tmp.QmUqfwqQo0 /tmp/tmp.GdnCpQ7Vka
++ return 0
+ kubectl_bin logs percona-server-mongodb-operator-6fc6c78cc6-6ljx9
++ mktemp
+ local LAST_OUT=/tmp/tmp.puOf4f7D06
++ mktemp
+ local LAST_ERR=/tmp/tmp.5JQkipac31
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs percona-server-mongodb-operator-6fc6c78cc6-6ljx9
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.puOf4f7D06
+ cat /tmp/tmp.5JQkipac31
+ rm /tmp/tmp.puOf4f7D06 /tmp/tmp.5JQkipac31
+ return 0
2025-06-16T02:49:50.581Z	INFO	setup	Manager starting up	{"gitCommit": "c5cc2da48d532d3f8efbbe61ee4d96d422a602b3", "gitBranch": "PR-1969-c5cc2da4", "buildTime": "", "goVersion": "go1.24.4", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-fs-25161
+ local namespace=demand-backup-fs-25161
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces demand-backup-fs-25161'
+ set +o xtrace

++ mktemp

-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-fs-25161
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-fs-25161 --ignore-not-found
+ local LAST_OUT=/tmp/tmp.AjAGZ0KruN
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.O8qsUyvWnp
+ local LAST_ERR=/tmp/tmp.VzvjqZb9Z0
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.wG3FeA1gkT
+ local exit_status=0
+ local timeout=4
+ for i in '$(seq 0 2)'
+ set +e
++ seq 0 2
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-fs-25161 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AjAGZ0KruN
+ cat /tmp/tmp.VzvjqZb9Z0
+ rm /tmp/tmp.AjAGZ0KruN /tmp/tmp.VzvjqZb9Z0
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.O8qsUyvWnp
+ cat /tmp/tmp.wG3FeA1gkT
+ rm /tmp/tmp.O8qsUyvWnp /tmp/tmp.wG3FeA1gkT
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-fs-25161
++ mktemp
+ local LAST_OUT=/tmp/tmp.3jaxU1IvY6
++ mktemp
+ local LAST_ERR=/tmp/tmp.mMLAk2INHu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace demand-backup-fs-25161
namespace "gke-managed-cim" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3jaxU1IvY6
+ cat /tmp/tmp.mMLAk2INHu
+ rm /tmp/tmp.3jaxU1IvY6 /tmp/tmp.mMLAk2INHu
+ return 0
+ desc 'create namespace demand-backup-fs-25161'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-fs-25161
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-fs-25161
++ mktemp
+ local LAST_OUT=/tmp/tmp.8btiBPcnQP
++ mktemp
+ local LAST_ERR=/tmp/tmp.zL3rcJrIiP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace demand-backup-fs-25161
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8btiBPcnQP
namespace/demand-backup-fs-25161 created
+ cat /tmp/tmp.zL3rcJrIiP
+ rm /tmp/tmp.8btiBPcnQP /tmp/tmp.zL3rcJrIiP
+ return 0
+ set_kube_ctx demand-backup-fs-25161
+ local namespace=demand-backup-fs-25161
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cK5NZcqzL4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.36o38UCMfS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cK5NZcqzL4
++ cat /tmp/tmp.36o38UCMfS
++ rm /tmp/tmp.cK5NZcqzL4 /tmp/tmp.36o38UCMfS
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1969-c5cc2da4-1-cluster4 --namespace=demand-backup-fs-25161
++ mktemp
+ local LAST_OUT=/tmp/tmp.qenjtcOtXJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.Wg09rT8gif
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1969-c5cc2da4-1-cluster4 --namespace=demand-backup-fs-25161
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qenjtcOtXJ
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1969-c5cc2da4-1-cluster4" modified.
+ cat /tmp/tmp.Wg09rT8gif
+ rm /tmp/tmp.qenjtcOtXJ /tmp/tmp.Wg09rT8gif
+ return 0
+ kubectl_bin delete ns storage
++ mktemp
+ local LAST_OUT=/tmp/tmp.yKMDKzdbjt
++ mktemp
+ local LAST_ERR=/tmp/tmp.9b0VxJu5Nw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ns storage
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.yKMDKzdbjt
+ cat /tmp/tmp.9b0VxJu5Nw
Error from server (NotFound): namespaces "storage" not found
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ns storage
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.yKMDKzdbjt
+ cat /tmp/tmp.9b0VxJu5Nw
Error from server (NotFound): namespaces "storage" not found
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ns storage
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.yKMDKzdbjt
+ cat /tmp/tmp.9b0VxJu5Nw
Error from server (NotFound): namespaces "storage" not found
+ sleep 8
+ cat /tmp/tmp.yKMDKzdbjt
+ cat /tmp/tmp.9b0VxJu5Nw
Error from server (NotFound): namespaces "storage" not found
+ rm /tmp/tmp.yKMDKzdbjt /tmp/tmp.9b0VxJu5Nw
+ return 1
+ :
+ [[ 1 != 1 ]]
+ uid=1001
+ [[ -n '' ]]
+ log 'deploying NFS server'
+ set +o xtrace
[2025-06-16T02:50:22+0000] deploying NFS server
+ deploy_nfs_server 1001
+ local uid=1001
+ kubectl_bin create namespace storage
++ mktemp
+ local LAST_OUT=/tmp/tmp.WfAVVxKU6a
++ mktemp
+ local LAST_ERR=/tmp/tmp.uiGxAw5EPm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace storage
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WfAVVxKU6a
namespace/storage created
+ cat /tmp/tmp.uiGxAw5EPm
+ rm /tmp/tmp.WfAVVxKU6a /tmp/tmp.uiGxAw5EPm
+ return 0
+ kubectl_bin apply -n storage -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/nfs-server.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.C5bDSO8cmB
++ mktemp
+ local LAST_ERR=/tmp/tmp.5U4EaZmI4k
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n storage -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/nfs-server.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C5bDSO8cmB
serviceaccount/nfs-server created
rolebinding.rbac.authorization.k8s.io/system:openshift:scc:privileged created
persistentvolumeclaim/nfs-pvc created
deployment.apps/nfs-server created
service/nfs-service created
+ cat /tmp/tmp.5U4EaZmI4k
+ rm /tmp/tmp.C5bDSO8cmB /tmp/tmp.5U4EaZmI4k
+ return 0
+ sleep 5
++ kubectl_bin get pod -n storage -l app=nfs-server -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cz2kuagQQ9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i39QKvMzh9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod -n storage -l app=nfs-server -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cz2kuagQQ9
++ cat /tmp/tmp.i39QKvMzh9
++ rm /tmp/tmp.cz2kuagQQ9 /tmp/tmp.i39QKvMzh9
++ return 0
+ local nfsPod=nfs-server-b799cbd49-htjsf
++ kubectl_bin get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5M96Vhgn73
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TEaGQ6oO7U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5M96Vhgn73
++ cat /tmp/tmp.TEaGQ6oO7U
++ rm /tmp/tmp.5M96Vhgn73 /tmp/tmp.TEaGQ6oO7U
++ return 0
+ [[ Pending == \R\u\n\n\i\n\g ]]
+ log 'Waiting for nfs-server-b799cbd49-htjsf to start Running'
+ set +o xtrace
[2025-06-16T02:50:33+0000] Waiting for nfs-server-b799cbd49-htjsf to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fFHMPq7Ffc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Tf2q5g01WU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fFHMPq7Ffc
++ cat /tmp/tmp.Tf2q5g01WU
++ rm /tmp/tmp.fFHMPq7Ffc /tmp/tmp.Tf2q5g01WU
++ return 0
+ [[ Pending == \R\u\n\n\i\n\g ]]
+ log 'Waiting for nfs-server-b799cbd49-htjsf to start Running'
+ set +o xtrace
[2025-06-16T02:50:35+0000] Waiting for nfs-server-b799cbd49-htjsf to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kUHg3PjO2P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Mf0saBKvn7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kUHg3PjO2P
++ cat /tmp/tmp.Mf0saBKvn7
++ rm /tmp/tmp.kUHg3PjO2P /tmp/tmp.Mf0saBKvn7
++ return 0
+ [[ Pending == \R\u\n\n\i\n\g ]]
+ log 'Waiting for nfs-server-b799cbd49-htjsf to start Running'
+ set +o xtrace
[2025-06-16T02:50:37+0000] Waiting for nfs-server-b799cbd49-htjsf to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H06o8W5Xd6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.My5d5Soe4E
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.H06o8W5Xd6
++ cat /tmp/tmp.My5d5Soe4E
++ rm /tmp/tmp.H06o8W5Xd6 /tmp/tmp.My5d5Soe4E
++ return 0
+ [[ Pending == \R\u\n\n\i\n\g ]]
+ log 'Waiting for nfs-server-b799cbd49-htjsf to start Running'
+ set +o xtrace
[2025-06-16T02:50:39+0000] Waiting for nfs-server-b799cbd49-htjsf to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KfjJjmRlEp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IwuzGXg6CM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KfjJjmRlEp
++ cat /tmp/tmp.IwuzGXg6CM
++ rm /tmp/tmp.KfjJjmRlEp /tmp/tmp.IwuzGXg6CM
++ return 0
+ [[ Pending == \R\u\n\n\i\n\g ]]
+ log 'Waiting for nfs-server-b799cbd49-htjsf to start Running'
+ set +o xtrace
[2025-06-16T02:50:41+0000] Waiting for nfs-server-b799cbd49-htjsf to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D5N0nNYL7D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bk4XMfXj0Y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D5N0nNYL7D
++ cat /tmp/tmp.bk4XMfXj0Y
++ rm /tmp/tmp.D5N0nNYL7D /tmp/tmp.bk4XMfXj0Y
++ return 0
+ [[ Pending == \R\u\n\n\i\n\g ]]
+ log 'Waiting for nfs-server-b799cbd49-htjsf to start Running'
+ set +o xtrace
[2025-06-16T02:50:43+0000] Waiting for nfs-server-b799cbd49-htjsf to start Running
+ sleep 1
++ kubectl_bin get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lDrbyqnOuF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BYPM3g105g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pod nfs-server-b799cbd49-htjsf -n storage -o 'jsonpath={.status.phase}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lDrbyqnOuF
++ cat /tmp/tmp.BYPM3g105g
++ rm /tmp/tmp.lDrbyqnOuF /tmp/tmp.BYPM3g105g
++ return 0
+ [[ Running == \R\u\n\n\i\n\g ]]
+ kubectl_bin exec -n storage nfs-server-b799cbd49-htjsf -- mkdir /exports/psmdb-some-name-rs0
++ mktemp
+ local LAST_OUT=/tmp/tmp.wF1wTxGVTg
++ mktemp
+ local LAST_ERR=/tmp/tmp.6Axd4JZrpH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec -n storage nfs-server-b799cbd49-htjsf -- mkdir /exports/psmdb-some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wF1wTxGVTg
+ cat /tmp/tmp.6Axd4JZrpH
+ rm /tmp/tmp.wF1wTxGVTg /tmp/tmp.6Axd4JZrpH
+ return 0
+ kubectl_bin exec -n storage nfs-server-b799cbd49-htjsf -- chown 1001:1001 /exports/psmdb-some-name-rs0
++ mktemp
+ local LAST_OUT=/tmp/tmp.YgKZ2E9UCH
++ mktemp
+ local LAST_ERR=/tmp/tmp.35xDPpDvJV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec -n storage nfs-server-b799cbd49-htjsf -- chown 1001:1001 /exports/psmdb-some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YgKZ2E9UCH
+ cat /tmp/tmp.35xDPpDvJV
+ rm /tmp/tmp.YgKZ2E9UCH /tmp/tmp.35xDPpDvJV
+ return 0
+ log 'creating secrets and start client'
+ set +o xtrace
[2025-06-16T02:50:47+0000] creating secrets and start client
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.A6yeMQwhqC
++ mktemp
+ local LAST_ERR=/tmp/tmp.6vd4rWJe9F
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A6yeMQwhqC
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.6vd4rWJe9F
+ rm /tmp/tmp.A6yeMQwhqC /tmp/tmp.6vd4rWJe9F
+ return 0
+ [[ -n '' ]]
+ log 'creating PSMDB cluster some-name'
+ set +o xtrace
[2025-06-16T02:50:49+0000] creating PSMDB cluster some-name
+ [[ 1 != 1 ]]
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/some-name.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1969-c5cc2da4"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.luI1P9DsV4
++ mktemp
+ local LAST_ERR=/tmp/tmp.rxFvOWxurU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.luI1P9DsV4
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.rxFvOWxurU
+ rm /tmp/tmp.luI1P9DsV4 /tmp/tmp.rxFvOWxurU
+ return 0
+ log 'wait for all 3 pods to start'
+ set +o xtrace
[2025-06-16T02:50:52+0000] wait for all 3 pods to start
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready..........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uCKdTQGyiW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.c0hwiINALj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uCKdTQGyiW
++ cat /tmp/tmp.c0hwiINALj
++ rm /tmp/tmp.uCKdTQGyiW /tmp/tmp.c0hwiINALj
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BhHixj7RiP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m4klEMYsOI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BhHixj7RiP
++ cat /tmp/tmp.m4klEMYsOI
++ rm /tmp/tmp.BhHixj7RiP /tmp/tmp.m4klEMYsOI
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5WDqUCUOf0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qLKmgqtXlA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5WDqUCUOf0
++ cat /tmp/tmp.qLKmgqtXlA
++ rm /tmp/tmp.5WDqUCUOf0 /tmp/tmp.qLKmgqtXlA
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..
+ [[ 1 == 1 ]]
+ log 'checking if statefulset created with expected config'
+ set +o xtrace
[2025-06-16T02:52:27+0000] checking if statefulset created with expected config
+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.JnbyHFcDhi/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-fs-25161", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.QIwVS6AkZX
++ mktemp
+ local LAST_ERR=/tmp/tmp.rh76s04eO4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QIwVS6AkZX
+ cat /tmp/tmp.rh76s04eO4
+ rm /tmp/tmp.QIwVS6AkZX /tmp/tmp.rh76s04eO4
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.JnbyHFcDhi/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.JnbyHFcDhi/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.JnbyHFcDhi/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/statefulset_some-name-rs0.yml /tmp/tmp.JnbyHFcDhi/statefulset_some-name-rs0.yml
+ log 'creating user'
+ set +o xtrace
[2025-06-16T02:52:28+0000] creating user
+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-fs-25161
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.A5T7iIpdEP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.56JFCtxOAi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.A5T7iIpdEP
++ cat /tmp/tmp.56JFCtxOAi
++ rm /tmp/tmp.A5T7iIpdEP /tmp/tmp.56JFCtxOAi
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.oDhAv4DfMs
++ mktemp
+ local LAST_ERR=/tmp/tmp.iV8omuxHWF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oDhAv4DfMs
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("aa03f059-4979-4f03-af37-c3fe7f21fe46") }
Percona Server for MongoDB server version: v8.0.8-3
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.iV8omuxHWF
+ rm /tmp/tmp.oDhAv4DfMs /tmp/tmp.iV8omuxHWF
+ return 0
+ sleep 2
+ log 'write initial data'
+ set +o xtrace
[2025-06-16T02:52:32+0000] write initial data
+ write_data 100500 ''
+ local x=100500
+ local find_prefix=
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bRt8y3qINp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zjnmRv8iTK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bRt8y3qINp
++ cat /tmp/tmp.zjnmRv8iTK
++ rm /tmp/tmp.bRt8y3qINp /tmp/tmp.zjnmRv8iTK
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.bOUVCidAv0
++ mktemp
+ local LAST_ERR=/tmp/tmp.vfZdmiF72U
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bOUVCidAv0
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("a7a28063-2d0a-4e86-b0af-31b7d0662b46") }
Percona Server for MongoDB server version: v8.0.8-3
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.vfZdmiF72U
+ rm /tmp/tmp.bOUVCidAv0 /tmp/tmp.vfZdmiF72U
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-25161 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-06-16T02:52:35+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-25161 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nUmssbVYpu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4eSoQfvg8y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nUmssbVYpu
++ cat /tmp/tmp.4eSoQfvg8y
++ rm /tmp/tmp.nUmssbVYpu /tmp/tmp.4eSoQfvg8y
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.k4e16JyYeO
++ mktemp
+ local LAST_ERR=/tmp/tmp.dY5rVc9gvK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.k4e16JyYeO
+ cat /tmp/tmp.dY5rVc9gvK
+ rm /tmp/tmp.k4e16JyYeO /tmp/tmp.dY5rVc9gvK
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/find.json /tmp/tmp.JnbyHFcDhi/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-06-16T02:52:22.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-06-16T02:52:27.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-06-16T02:52:29.000+0000 I listening for the commands

+ desc 'CASE 1: Logical backup and restore'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 1: Logical backup and restore
-----------------------------------------------------------------------------------

+ backup_name=backup-nfs-logical
+ run_backup nfs backup-nfs-logical logical
+ local storage=nfs
+ local backup_name=backup-nfs-logical
+ local type=logical
+ desc 'run backup backup-nfs-logical'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-nfs-logical
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f -
+ yq eval '.metadata.name = "backup-nfs-logical"
			| .spec.storageName = "nfs"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/backup-nfs.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.RFOhUTF13d
++ mktemp
+ local LAST_ERR=/tmp/tmp.znfhoNzQ5t
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RFOhUTF13d
perconaservermongodbbackup.psmdb.percona.com/backup-nfs-logical created
+ cat /tmp/tmp.znfhoNzQ5t
+ rm /tmp/tmp.RFOhUTF13d /tmp/tmp.znfhoNzQ5t
+ return 0
+ wait_backup backup-nfs-logical
+ local backup_name=backup-nfs-logical
+ local target_state=ready
+ set +o xtrace
waiting for backup-nfs-logical to reach ready state......
+ run_recovery_check backup-nfs-logical some-name -2nd ''
+ local backup=backup-nfs-logical
+ local cluster=some-name
+ local find_prefix_before=-2nd
+ local find_prefix_after=
+ write_data 100501 -2nd
+ local x=100501
+ local find_prefix=-2nd
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Hg6ZCaYsJo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5lokK2j4ts
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Hg6ZCaYsJo
++ cat /tmp/tmp.5lokK2j4ts
++ rm /tmp/tmp.Hg6ZCaYsJo /tmp/tmp.5lokK2j4ts
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pRQEuMHXJs
++ mktemp
+ local LAST_ERR=/tmp/tmp.AFcJicEraZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pRQEuMHXJs
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("bb459052-c15b-410a-8878-4e693739530e") }
Percona Server for MongoDB server version: v8.0.8-3
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.AFcJicEraZ
+ rm /tmp/tmp.pRQEuMHXJs /tmp/tmp.AFcJicEraZ
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-25161 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-06-16T02:52:55+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-25161 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Llrkgktmwb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PkTAmQyTcb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Llrkgktmwb
++ cat /tmp/tmp.PkTAmQyTcb
++ rm /tmp/tmp.Llrkgktmwb /tmp/tmp.PkTAmQyTcb
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3JtJDqjtsz
++ mktemp
+ local LAST_ERR=/tmp/tmp.cdI83vCGpU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3JtJDqjtsz
+ cat /tmp/tmp.cdI83vCGpU
+ rm /tmp/tmp.3JtJDqjtsz /tmp/tmp.cdI83vCGpU
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/find-2nd.json /tmp/tmp.JnbyHFcDhi/find-2nd
+ run_restore backup-nfs-logical
+ local backup_name=backup-nfs-logical
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-nfs-logical/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-nfs-logical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.YDyvfSM9Q2
++ mktemp
+ local LAST_ERR=/tmp/tmp.mmJkyYmWJz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YDyvfSM9Q2
perconaservermongodbrestore.psmdb.percona.com/restore-backup-nfs-logical created
+ cat /tmp/tmp.mmJkyYmWJz
+ rm /tmp/tmp.YDyvfSM9Q2 /tmp/tmp.mmJkyYmWJz
+ return 0
+ wait_restore backup-nfs-logical some-name
+ local backup_name=backup-nfs-logical
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-nfs-logical object to be createdOK
Waiting psmdb-restore/restore-backup-nfs-logical to reach state "ready" OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EbEI7etdK7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4IZhqe1iLi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EbEI7etdK7
++ cat /tmp/tmp.4IZhqe1iLi
++ rm /tmp/tmp.EbEI7etdK7 /tmp/tmp.4IZhqe1iLi
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-25161 '' .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-06-16T02:53:33+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-25161 mongodb .svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4XyL53PNsX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DiiJAXRCQ2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4XyL53PNsX
++ cat /tmp/tmp.DiiJAXRCQ2
++ rm /tmp/tmp.4XyL53PNsX /tmp/tmp.DiiJAXRCQ2
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ChnILTV3SE
++ mktemp
+ local LAST_ERR=/tmp/tmp.yyrJfhuohm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ChnILTV3SE
+ cat /tmp/tmp.yyrJfhuohm
+ rm /tmp/tmp.ChnILTV3SE /tmp/tmp.yyrJfhuohm
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/find.json /tmp/tmp.JnbyHFcDhi/find
+ desc 'CASE 2: Logical backup and PiTR'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 2: Logical backup and PiTR
-----------------------------------------------------------------------------------

+ backup_name=backup-nfs-logical-pitr
+ run_backup nfs backup-nfs-logical-pitr logical
+ local storage=nfs
+ local backup_name=backup-nfs-logical-pitr
+ local type=logical
+ desc 'run backup backup-nfs-logical-pitr'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-nfs-logical-pitr
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-nfs-logical-pitr"
			| .spec.storageName = "nfs"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/backup-nfs.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.wtm4PSsfBl
++ mktemp
+ local LAST_ERR=/tmp/tmp.6n0o3kIdyG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wtm4PSsfBl
perconaservermongodbbackup.psmdb.percona.com/backup-nfs-logical-pitr created
+ cat /tmp/tmp.6n0o3kIdyG
+ rm /tmp/tmp.wtm4PSsfBl /tmp/tmp.6n0o3kIdyG
+ return 0
+ wait_backup backup-nfs-logical-pitr
+ local backup_name=backup-nfs-logical-pitr
+ local target_state=ready
+ set +o xtrace
waiting for backup-nfs-logical-pitr to reach ready state......
+ write_data 100502 -3rd
+ local x=100502
+ local find_prefix=-3rd
+ run_mongo 'use myApp\n db.test.insert({ x: 100502 })' myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local 'command=use myApp\n db.test.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1x3Zd3P2gy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7OPnDh8Wva
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1x3Zd3P2gy
++ cat /tmp/tmp.7OPnDh8Wva
++ rm /tmp/tmp.1x3Zd3P2gy /tmp/tmp.7OPnDh8Wva
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Zup7u05LfR
++ mktemp
+ local LAST_ERR=/tmp/tmp.EhxJtfkDyZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Zup7u05LfR
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("19d3e934-33de-4f9c-9031-5958e8f48eae") }
Percona Server for MongoDB server version: v8.0.8-3
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.EhxJtfkDyZ
+ rm /tmp/tmp.Zup7u05LfR /tmp/tmp.EhxJtfkDyZ
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-25161 -3rd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local postfix=-3rd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-06-16T02:53:53+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-25161 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rcbLnB7asF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UZOifhyRIT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rcbLnB7asF
++ cat /tmp/tmp.UZOifhyRIT
++ rm /tmp/tmp.rcbLnB7asF /tmp/tmp.UZOifhyRIT
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.MpjbuZyu3O
++ mktemp
+ local LAST_ERR=/tmp/tmp.aqINTYo2Ko
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MpjbuZyu3O
+ cat /tmp/tmp.aqINTYo2Ko
+ rm /tmp/tmp.MpjbuZyu3O /tmp/tmp.aqINTYo2Ko
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/find-3rd.json /tmp/tmp.JnbyHFcDhi/find-3rd
+ run_pitr_check backup-nfs-logical-pitr some-name -3rd
+ local backup=backup-nfs-logical-pitr
+ local cluster=some-name
+ local find_prefix=-3rd
+ wait_for_oplogs some-name
+ local cluster1=some-name
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ jq '.backups.snapshot[0].restoreTo'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oRptf1BXjV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IIB51tvS67
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oRptf1BXjV
++ cat /tmp/tmp.IIB51tvS67
++ rm /tmp/tmp.oRptf1BXjV /tmp/tmp.IIB51tvS67
++ return 0
+ local backup_last_write=1750042422
+ local retries=0
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.0i3OMLjRNC
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.hYlD07TMmU
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.0i3OMLjRNC
+++ cat /tmp/tmp.hYlD07TMmU
+++ rm /tmp/tmp.0i3OMLjRNC /tmp/tmp.hYlD07TMmU
+++ return 0
++ echo 1750042389
+ local last_chunk=1750042389
+ [[ 1750042389 -gt 1750042422 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.skkXWgluDs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3glbh8XMi4
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.skkXWgluDs
+++ cat /tmp/tmp.3glbh8XMi4
+++ rm /tmp/tmp.skkXWgluDs /tmp/tmp.3glbh8XMi4
+++ return 0
++ echo 1750042389
+ last_chunk=1750042389
+ retries=1
++ format_date 1750042389
++ local timestamp=1750042389
+++ TZ=UTC
+++ /usr/bin/date -d@1750042389 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:09
++ format_date 1750042422
++ local timestamp=1750042422
+++ TZ=UTC
+++ /usr/bin/date -d@1750042422 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:42
+ log 'Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)'
+ set +o xtrace
[2025-06-16T02:54:00+0000] Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)
+ sleep 10
+ [[ 1750042389 -gt 1750042422 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SwGxmXSSPt
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iRw1JPbBZW
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.SwGxmXSSPt
+++ cat /tmp/tmp.iRw1JPbBZW
+++ rm /tmp/tmp.SwGxmXSSPt /tmp/tmp.iRw1JPbBZW
+++ return 0
++ echo 1750042389
+ last_chunk=1750042389
+ retries=2
++ format_date 1750042389
++ local timestamp=1750042389
+++ TZ=UTC
+++ /usr/bin/date -d@1750042389 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:09
++ format_date 1750042422
++ local timestamp=1750042422
+++ TZ=UTC
+++ /usr/bin/date -d@1750042422 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:42
+ log 'Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)'
+ set +o xtrace
[2025-06-16T02:54:12+0000] Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)
+ sleep 10
+ [[ 1750042389 -gt 1750042422 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.mDpDIAtlfc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fQAC4eEQ5i
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.mDpDIAtlfc
+++ cat /tmp/tmp.fQAC4eEQ5i
+++ rm /tmp/tmp.mDpDIAtlfc /tmp/tmp.fQAC4eEQ5i
+++ return 0
++ echo 1750042389
+ last_chunk=1750042389
+ retries=3
++ format_date 1750042389
++ local timestamp=1750042389
+++ TZ=UTC
+++ /usr/bin/date -d@1750042389 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:09
++ format_date 1750042422
++ local timestamp=1750042422
+++ TZ=UTC
+++ /usr/bin/date -d@1750042422 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:42
+ log 'Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)'
+ set +o xtrace
[2025-06-16T02:54:24+0000] Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)
+ sleep 10
+ [[ 1750042389 -gt 1750042422 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1cFJM3V1ge
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.U9clQ0ereU
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.1cFJM3V1ge
+++ cat /tmp/tmp.U9clQ0ereU
+++ rm /tmp/tmp.1cFJM3V1ge /tmp/tmp.U9clQ0ereU
+++ return 0
++ echo 1750042389
+ last_chunk=1750042389
+ retries=4
++ format_date 1750042389
++ local timestamp=1750042389
+++ TZ=UTC
+++ /usr/bin/date -d@1750042389 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:09
++ format_date 1750042422
++ local timestamp=1750042422
+++ TZ=UTC
+++ /usr/bin/date -d@1750042422 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:42
+ log 'Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)'
+ set +o xtrace
[2025-06-16T02:54:36+0000] Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)
+ sleep 10
+ [[ 1750042389 -gt 1750042422 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.VsajUmZObo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6utEF8ZWcr
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.VsajUmZObo
+++ cat /tmp/tmp.6utEF8ZWcr
+++ rm /tmp/tmp.VsajUmZObo /tmp/tmp.6utEF8ZWcr
+++ return 0
++ echo 1750042389
+ last_chunk=1750042389
+ retries=5
++ format_date 1750042389
++ local timestamp=1750042389
+++ TZ=UTC
+++ /usr/bin/date -d@1750042389 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:09
++ format_date 1750042422
++ local timestamp=1750042422
+++ TZ=UTC
+++ /usr/bin/date -d@1750042422 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:42
+ log 'Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)'
+ set +o xtrace
[2025-06-16T02:54:47+0000] Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)
+ sleep 10
+ [[ 1750042389 -gt 1750042422 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.rVGWk2hynH
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kNPBb9XiaC
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.rVGWk2hynH
+++ cat /tmp/tmp.kNPBb9XiaC
+++ rm /tmp/tmp.rVGWk2hynH /tmp/tmp.kNPBb9XiaC
+++ return 0
++ echo 1750042389
+ last_chunk=1750042389
+ retries=6
++ format_date 1750042389
++ local timestamp=1750042389
+++ TZ=UTC
+++ /usr/bin/date -d@1750042389 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:09
++ format_date 1750042422
++ local timestamp=1750042422
+++ TZ=UTC
+++ /usr/bin/date -d@1750042422 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:42
+ log 'Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)'
+ set +o xtrace
[2025-06-16T02:54:59+0000] Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)
+ sleep 10
+ [[ 1750042389 -gt 1750042422 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.lxGDFsYkIy
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.o7MH773eNb
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.lxGDFsYkIy
+++ cat /tmp/tmp.o7MH773eNb
+++ rm /tmp/tmp.lxGDFsYkIy /tmp/tmp.o7MH773eNb
+++ return 0
++ echo 1750042389
+ last_chunk=1750042389
+ retries=7
++ format_date 1750042389
++ local timestamp=1750042389
+++ TZ=UTC
+++ /usr/bin/date -d@1750042389 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:09
++ format_date 1750042422
++ local timestamp=1750042422
+++ TZ=UTC
+++ /usr/bin/date -d@1750042422 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:42
+ log 'Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)'
+ set +o xtrace
[2025-06-16T02:55:11+0000] Waiting for last oplog chunk (2025-06-16 02:53:09) to be greater than last write (2025-06-16 02:53:42)
+ sleep 10
+ [[ 1750042389 -gt 1750042422 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9CafeC13CQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6dQRfTWVUD
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.9CafeC13CQ
+++ cat /tmp/tmp.6dQRfTWVUD
+++ rm /tmp/tmp.9CafeC13CQ /tmp/tmp.6dQRfTWVUD
+++ return 0
++ echo 1750042512
+ last_chunk=1750042512
+ retries=8
++ format_date 1750042512
++ local timestamp=1750042512
+++ TZ=UTC
+++ /usr/bin/date -d@1750042512 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:55:12
++ format_date 1750042422
++ local timestamp=1750042422
+++ TZ=UTC
+++ /usr/bin/date -d@1750042422 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:53:42
+ log 'Waiting for last oplog chunk (2025-06-16 02:55:12) to be greater than last write (2025-06-16 02:53:42)'
+ set +o xtrace
[2025-06-16T02:55:23+0000] Waiting for last oplog chunk (2025-06-16 02:55:12) to be greater than last write (2025-06-16 02:53:42)
+ sleep 10
+ [[ 1750042512 -gt 1750042422 ]]
+++ get_latest_oplog_chunk_ts some-name
+++ local cluster=some-name
++++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.yXnrbvnPwS
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.Eagp649GPt
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.yXnrbvnPwS
++++ cat /tmp/tmp.Eagp649GPt
++++ rm /tmp/tmp.yXnrbvnPwS /tmp/tmp.Eagp649GPt
++++ return 0
+++ echo 1750042512
++ format_date 1750042512
++ local timestamp=1750042512
+++ TZ=UTC
+++ /usr/bin/date -d@1750042512 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:55:12
+ local 'target_time=2025-06-16 02:55:12'
+ log 'dropping test collection'
+ set +o xtrace
[2025-06-16T02:55:34+0000] dropping test collection
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5GDIRAS3ih
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TMBYKC85s6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5GDIRAS3ih
++ cat /tmp/tmp.TMBYKC85s6
++ rm /tmp/tmp.5GDIRAS3ih /tmp/tmp.TMBYKC85s6
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DMlVAcooWs
++ mktemp
+ local LAST_ERR=/tmp/tmp.LVqJLbe5CQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DMlVAcooWs
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8d9babf6-7a9c-4194-b07b-74d81dcf7733") }
Percona Server for MongoDB server version: v8.0.8-3
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.LVqJLbe5CQ
+ rm /tmp/tmp.DMlVAcooWs /tmp/tmp.LVqJLbe5CQ
+ return 0
+ log 'checking pitr... backup: backup-nfs-logical-pitr target: 2025-06-16 02:55:12'
+ set +o xtrace
[2025-06-16T02:55:37+0000] checking pitr... backup: backup-nfs-logical-pitr target: 2025-06-16 02:55:12
+ yq eval '.metadata.name = "restore-backup-nfs-logical-pitr"'
+ yq eval '.spec.backupName = "backup-nfs-logical-pitr"'
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/pitr.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.NTieZrfB6K
++ mktemp
+ local LAST_ERR=/tmp/tmp.metPd8o3HM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ yq eval '.spec.pitr.date = "2025-06-16 02:55:12"'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NTieZrfB6K
perconaservermongodbrestore.psmdb.percona.com/restore-backup-nfs-logical-pitr created
+ cat /tmp/tmp.metPd8o3HM
+ rm /tmp/tmp.NTieZrfB6K /tmp/tmp.metPd8o3HM
+ return 0
+ wait_restore backup-nfs-logical-pitr some-name
+ local backup_name=backup-nfs-logical-pitr
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-nfs-logical-pitr object to be createdOK
Waiting psmdb-restore/restore-backup-nfs-logical-pitr to reach state "ready" OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.o1jjAPiCNH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hu6MT1VqBS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.o1jjAPiCNH
++ cat /tmp/tmp.hu6MT1VqBS
++ rm /tmp/tmp.o1jjAPiCNH /tmp/tmp.hu6MT1VqBS
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-25161 -3rd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local postfix=-3rd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-06-16T02:56:04+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-25161 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TQE2SltDkF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IQXI9UUDwO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TQE2SltDkF
++ cat /tmp/tmp.IQXI9UUDwO
++ rm /tmp/tmp.TQE2SltDkF /tmp/tmp.IQXI9UUDwO
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.N6mqnMUbYa
++ mktemp
+ local LAST_ERR=/tmp/tmp.wVKuzSgCGP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N6mqnMUbYa
+ cat /tmp/tmp.wVKuzSgCGP
+ rm /tmp/tmp.N6mqnMUbYa /tmp/tmp.wVKuzSgCGP
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/find-3rd.json /tmp/tmp.JnbyHFcDhi/find-3rd
+ desc 'CASE 3: Physical backup and restore'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 3: Physical backup and restore
-----------------------------------------------------------------------------------

+ backup_name=backup-nfs-physical
+ run_backup nfs backup-nfs-physical physical
+ local storage=nfs
+ local backup_name=backup-nfs-physical
+ local type=physical
+ desc 'run backup backup-nfs-physical'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-nfs-physical
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-nfs-physical"
			| .spec.storageName = "nfs"
			| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/backup-nfs.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.jOB4PLgXSh
++ mktemp
+ local LAST_ERR=/tmp/tmp.X5Gm8V6XjI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jOB4PLgXSh
perconaservermongodbbackup.psmdb.percona.com/backup-nfs-physical created
+ cat /tmp/tmp.X5Gm8V6XjI
+ rm /tmp/tmp.jOB4PLgXSh /tmp/tmp.X5Gm8V6XjI
+ return 0
+ wait_backup backup-nfs-physical
+ local backup_name=backup-nfs-physical
+ local target_state=ready
+ set +o xtrace
waiting for backup-nfs-physical to reach ready state......
+ run_recovery_check backup-nfs-physical some-name -4th -3rd
+ local backup=backup-nfs-physical
+ local cluster=some-name
+ local find_prefix_before=-4th
+ local find_prefix_after=-3rd
+ write_data 100501 -4th
+ local x=100501
+ local find_prefix=-4th
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.esZHqwlFP3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HShpF5DFuW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.esZHqwlFP3
++ cat /tmp/tmp.HShpF5DFuW
++ rm /tmp/tmp.esZHqwlFP3 /tmp/tmp.HShpF5DFuW
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.eVCKRCb5bE
++ mktemp
+ local LAST_ERR=/tmp/tmp.y6K43HdjFN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eVCKRCb5bE
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("916ba27a-caef-4a49-b1bf-566630595f96") }
Percona Server for MongoDB server version: v8.0.8-3
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.y6K43HdjFN
+ rm /tmp/tmp.eVCKRCb5bE /tmp/tmp.y6K43HdjFN
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-25161 -4th .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local postfix=-4th
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-06-16T02:56:23+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-25161 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SiuIyMMeYG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SBeF0QEQYu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SiuIyMMeYG
++ cat /tmp/tmp.SBeF0QEQYu
++ rm /tmp/tmp.SiuIyMMeYG /tmp/tmp.SBeF0QEQYu
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.cnFT2o5KMP
++ mktemp
+ local LAST_ERR=/tmp/tmp.ktca7zWaL6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cnFT2o5KMP
+ cat /tmp/tmp.ktca7zWaL6
+ rm /tmp/tmp.cnFT2o5KMP /tmp/tmp.ktca7zWaL6
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/find-4th.json /tmp/tmp.JnbyHFcDhi/find-4th
+ run_restore backup-nfs-physical
+ local backup_name=backup-nfs-physical
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-nfs-physical/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-nfs-physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.vcZO2qCo1V
++ mktemp
+ local LAST_ERR=/tmp/tmp.zp8NAw7tQb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vcZO2qCo1V
perconaservermongodbrestore.psmdb.percona.com/restore-backup-nfs-physical created
+ cat /tmp/tmp.zp8NAw7tQb
+ rm /tmp/tmp.vcZO2qCo1V /tmp/tmp.zp8NAw7tQb
+ return 0
+ wait_restore backup-nfs-physical some-name
+ local backup_name=backup-nfs-physical
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-nfs-physical object to be createdOK
Waiting psmdb-restore/restore-backup-nfs-physical to reach state "ready" ...OK after 3 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SziKvB4rCo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DOLpC6fze2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SziKvB4rCo
++ cat /tmp/tmp.DOLpC6fze2
++ rm /tmp/tmp.SziKvB4rCo /tmp/tmp.DOLpC6fze2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z6DiufcbsT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5LRAANri18
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.z6DiufcbsT
++ cat /tmp/tmp.5LRAANri18
++ rm /tmp/tmp.z6DiufcbsT /tmp/tmp.5LRAANri18
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.owIoC7MEEM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PQMRbhWMtQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.owIoC7MEEM
++ cat /tmp/tmp.PQMRbhWMtQ
++ rm /tmp/tmp.owIoC7MEEM /tmp/tmp.PQMRbhWMtQ
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yEJ9bpnBXE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G9xR7uUyMv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yEJ9bpnBXE
++ cat /tmp/tmp.G9xR7uUyMv
++ rm /tmp/tmp.yEJ9bpnBXE /tmp/tmp.G9xR7uUyMv
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.itRbHGSHPR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DlGC1bEO6o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.itRbHGSHPR
++ cat /tmp/tmp.DlGC1bEO6o
++ rm /tmp/tmp.itRbHGSHPR /tmp/tmp.DlGC1bEO6o
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8p4K5drQBs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K5VfSM6Oh8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8p4K5drQBs
++ cat /tmp/tmp.K5VfSM6Oh8
++ rm /tmp/tmp.8p4K5drQBs /tmp/tmp.K5VfSM6Oh8
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mxARMrlzLP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5i0sfVCIle
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mxARMrlzLP
++ cat /tmp/tmp.5i0sfVCIle
++ rm /tmp/tmp.mxARMrlzLP /tmp/tmp.5i0sfVCIle
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.itZrVYINob
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MJfes2c7kh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.itZrVYINob
++ cat /tmp/tmp.MJfes2c7kh
++ rm /tmp/tmp.itZrVYINob /tmp/tmp.MJfes2c7kh
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gYMGo4QSbG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rs9SFeUFj3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gYMGo4QSbG
++ cat /tmp/tmp.rs9SFeUFj3
++ rm /tmp/tmp.gYMGo4QSbG /tmp/tmp.rs9SFeUFj3
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2gruMTrnzY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ugc6Qz4W7h
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2gruMTrnzY
++ cat /tmp/tmp.Ugc6Qz4W7h
++ rm /tmp/tmp.2gruMTrnzY /tmp/tmp.Ugc6Qz4W7h
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jAursUD1ym
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eJ3xBNqgUT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jAursUD1ym
++ cat /tmp/tmp.eJ3xBNqgUT
++ rm /tmp/tmp.jAursUD1ym /tmp/tmp.eJ3xBNqgUT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Bapzru4Fvv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WspRxDh20o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Bapzru4Fvv
++ cat /tmp/tmp.WspRxDh20o
++ rm /tmp/tmp.Bapzru4Fvv /tmp/tmp.WspRxDh20o
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GyCbigSRRh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Mr6T6xFFwR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GyCbigSRRh
++ cat /tmp/tmp.Mr6T6xFFwR
++ rm /tmp/tmp.GyCbigSRRh /tmp/tmp.Mr6T6xFFwR
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QdwudKB1xU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z1cuKiOYm9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QdwudKB1xU
++ cat /tmp/tmp.Z1cuKiOYm9
++ rm /tmp/tmp.QdwudKB1xU /tmp/tmp.Z1cuKiOYm9
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-25161 -3rd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local postfix=-3rd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-06-16T03:03:03+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-25161 mongodb .svc.cluster.local
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XU6hA600Xd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ChIku63d1p
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XU6hA600Xd
++ cat /tmp/tmp.ChIku63d1p
++ rm /tmp/tmp.XU6hA600Xd /tmp/tmp.ChIku63d1p
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lzf8BFMZAh
++ mktemp
+ local LAST_ERR=/tmp/tmp.9cLlmvbFiL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lzf8BFMZAh
+ cat /tmp/tmp.9cLlmvbFiL
+ rm /tmp/tmp.lzf8BFMZAh /tmp/tmp.9cLlmvbFiL
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/find-3rd.json /tmp/tmp.JnbyHFcDhi/find-3rd
+ desc 'CASE 4: Physical backup and PiTR'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 4: Physical backup and PiTR
-----------------------------------------------------------------------------------

+ backup_name=backup-nfs-physical-pitr
+ run_backup nfs backup-nfs-physical-pitr physical
+ local storage=nfs
+ local backup_name=backup-nfs-physical-pitr
+ local type=physical
+ desc 'run backup backup-nfs-physical-pitr'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-nfs-physical-pitr
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-nfs-physical-pitr"
			| .spec.storageName = "nfs"
			| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/backup-nfs.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Od90MwVjUO
++ mktemp
+ local LAST_ERR=/tmp/tmp.Iio7Tdl5g4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Od90MwVjUO
perconaservermongodbbackup.psmdb.percona.com/backup-nfs-physical-pitr created
+ cat /tmp/tmp.Iio7Tdl5g4
+ rm /tmp/tmp.Od90MwVjUO /tmp/tmp.Iio7Tdl5g4
+ return 0
+ wait_backup backup-nfs-physical-pitr
+ local backup_name=backup-nfs-physical-pitr
+ local target_state=ready
+ set +o xtrace
waiting for backup-nfs-physical-pitr to reach ready state......
+ write_data 100503 -5th
+ local x=100503
+ local find_prefix=-5th
+ run_mongo 'use myApp\n db.test.insert({ x: 100503 })' myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local 'command=use myApp\n db.test.insert({ x: 100503 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ddob7NVUdr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5Pdr6MTeaS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ddob7NVUdr
++ cat /tmp/tmp.5Pdr6MTeaS
++ rm /tmp/tmp.Ddob7NVUdr /tmp/tmp.5Pdr6MTeaS
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.s4Qss8yEPx
++ mktemp
+ local LAST_ERR=/tmp/tmp.Rqd4z7U2Du
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.s4Qss8yEPx
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("d5beebd9-e291-4591-b313-72233c3f203e") }
Percona Server for MongoDB server version: v8.0.8-3
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Rqd4z7U2Du
+ rm /tmp/tmp.s4Qss8yEPx /tmp/tmp.Rqd4z7U2Du
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-25161 -5th .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local postfix=-5th
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-06-16T03:03:23+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-25161 mongodb .svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oPEYIbR0HQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k41lP2Te5J
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oPEYIbR0HQ
++ cat /tmp/tmp.k41lP2Te5J
++ rm /tmp/tmp.oPEYIbR0HQ /tmp/tmp.k41lP2Te5J
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.bGgopv75Mw
++ mktemp
+ local LAST_ERR=/tmp/tmp.acDo2k3x2a
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bGgopv75Mw
+ cat /tmp/tmp.acDo2k3x2a
+ rm /tmp/tmp.bGgopv75Mw /tmp/tmp.acDo2k3x2a
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/find-5th.json /tmp/tmp.JnbyHFcDhi/find-5th
+ run_pitr_check backup-nfs-physical-pitr some-name -5th
+ local backup=backup-nfs-physical-pitr
+ local cluster=some-name
+ local find_prefix=-5th
+ wait_for_oplogs some-name
+ local cluster1=some-name
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ mktemp
++ jq '.backups.snapshot[0].restoreTo'
++ local LAST_OUT=/tmp/tmp.PnFiTR0buL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PlMaLctl62
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PnFiTR0buL
++ cat /tmp/tmp.PlMaLctl62
++ rm /tmp/tmp.PnFiTR0buL /tmp/tmp.PlMaLctl62
++ return 0
+ local backup_last_write=1750042990
+ local retries=0
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kfsTQhVNJT
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.V69Agnelrx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.kfsTQhVNJT
+++ cat /tmp/tmp.V69Agnelrx
+++ rm /tmp/tmp.kfsTQhVNJT /tmp/tmp.V69Agnelrx
+++ return 0
++ echo 1750042542
+ local last_chunk=1750042542
+ [[ 1750042542 -gt 1750042990 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ZHhU1Kjkkv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fO5IQGQ4xd
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ZHhU1Kjkkv
+++ cat /tmp/tmp.fO5IQGQ4xd
+++ rm /tmp/tmp.ZHhU1Kjkkv /tmp/tmp.fO5IQGQ4xd
+++ return 0
++ echo 1750042542
+ last_chunk=1750042542
+ retries=1
++ format_date 1750042542
++ local timestamp=1750042542
+++ TZ=UTC
+++ /usr/bin/date -d@1750042542 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:55:42
++ format_date 1750042990
++ local timestamp=1750042990
+++ TZ=UTC
+++ /usr/bin/date -d@1750042990 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 03:03:10
+ log 'Waiting for last oplog chunk (2025-06-16 02:55:42) to be greater than last write (2025-06-16 03:03:10)'
+ set +o xtrace
[2025-06-16T03:03:30+0000] Waiting for last oplog chunk (2025-06-16 02:55:42) to be greater than last write (2025-06-16 03:03:10)
+ sleep 10
+ [[ 1750042542 -gt 1750042990 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.VwYapYXGzB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.mHm2S2c37h
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.VwYapYXGzB
+++ cat /tmp/tmp.mHm2S2c37h
+++ rm /tmp/tmp.VwYapYXGzB /tmp/tmp.mHm2S2c37h
+++ return 0
++ echo 1750042542
+ last_chunk=1750042542
+ retries=2
++ format_date 1750042542
++ local timestamp=1750042542
+++ TZ=UTC
+++ /usr/bin/date -d@1750042542 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:55:42
++ format_date 1750042990
++ local timestamp=1750042990
+++ TZ=UTC
+++ /usr/bin/date -d@1750042990 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 03:03:10
+ log 'Waiting for last oplog chunk (2025-06-16 02:55:42) to be greater than last write (2025-06-16 03:03:10)'
+ set +o xtrace
[2025-06-16T03:03:42+0000] Waiting for last oplog chunk (2025-06-16 02:55:42) to be greater than last write (2025-06-16 03:03:10)
+ sleep 10
+ [[ 1750042542 -gt 1750042990 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.RCFtza0hWg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1U2Q21T2Hb
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.RCFtza0hWg
+++ cat /tmp/tmp.1U2Q21T2Hb
+++ rm /tmp/tmp.RCFtza0hWg /tmp/tmp.1U2Q21T2Hb
+++ return 0
++ echo 1750042542
+ last_chunk=1750042542
+ retries=3
++ format_date 1750042542
++ local timestamp=1750042542
+++ TZ=UTC
+++ /usr/bin/date -d@1750042542 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:55:42
++ format_date 1750042990
++ local timestamp=1750042990
+++ TZ=UTC
+++ /usr/bin/date -d@1750042990 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 03:03:10
+ log 'Waiting for last oplog chunk (2025-06-16 02:55:42) to be greater than last write (2025-06-16 03:03:10)'
+ set +o xtrace
[2025-06-16T03:03:54+0000] Waiting for last oplog chunk (2025-06-16 02:55:42) to be greater than last write (2025-06-16 03:03:10)
+ sleep 10
+ [[ 1750042542 -gt 1750042990 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dBkk6z7jvC
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.XtLqvNaZCI
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.dBkk6z7jvC
+++ cat /tmp/tmp.XtLqvNaZCI
+++ rm /tmp/tmp.dBkk6z7jvC /tmp/tmp.XtLqvNaZCI
+++ return 0
++ echo 1750042542
+ last_chunk=1750042542
+ retries=4
++ format_date 1750042542
++ local timestamp=1750042542
+++ TZ=UTC
+++ /usr/bin/date -d@1750042542 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:55:42
++ format_date 1750042990
++ local timestamp=1750042990
+++ TZ=UTC
+++ /usr/bin/date -d@1750042990 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 03:03:10
+ log 'Waiting for last oplog chunk (2025-06-16 02:55:42) to be greater than last write (2025-06-16 03:03:10)'
+ set +o xtrace
[2025-06-16T03:04:05+0000] Waiting for last oplog chunk (2025-06-16 02:55:42) to be greater than last write (2025-06-16 03:03:10)
+ sleep 10
+ [[ 1750042542 -gt 1750042990 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9CU3KytsT0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vm0fxzYHXx
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.9CU3KytsT0
+++ cat /tmp/tmp.vm0fxzYHXx
+++ rm /tmp/tmp.9CU3KytsT0 /tmp/tmp.vm0fxzYHXx
+++ return 0
++ echo 1750042542
+ last_chunk=1750042542
+ retries=5
++ format_date 1750042542
++ local timestamp=1750042542
+++ TZ=UTC
+++ /usr/bin/date -d@1750042542 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:55:42
++ format_date 1750042990
++ local timestamp=1750042990
+++ TZ=UTC
+++ /usr/bin/date -d@1750042990 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 03:03:10
+ log 'Waiting for last oplog chunk (2025-06-16 02:55:42) to be greater than last write (2025-06-16 03:03:10)'
+ set +o xtrace
[2025-06-16T03:04:17+0000] Waiting for last oplog chunk (2025-06-16 02:55:42) to be greater than last write (2025-06-16 03:03:10)
+ sleep 10
+ [[ 1750042542 -gt 1750042990 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.mHL2JdHMus
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.El9bdKMotS
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.mHL2JdHMus
+++ cat /tmp/tmp.El9bdKMotS
+++ rm /tmp/tmp.mHL2JdHMus /tmp/tmp.El9bdKMotS
+++ return 0
++ echo 1750042542
+ last_chunk=1750042542
+ retries=6
++ format_date 1750042542
++ local timestamp=1750042542
+++ TZ=UTC
+++ /usr/bin/date -d@1750042542 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 02:55:42
++ format_date 1750042990
++ local timestamp=1750042990
+++ TZ=UTC
+++ /usr/bin/date -d@1750042990 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 03:03:10
+ log 'Waiting for last oplog chunk (2025-06-16 02:55:42) to be greater than last write (2025-06-16 03:03:10)'
+ set +o xtrace
[2025-06-16T03:04:29+0000] Waiting for last oplog chunk (2025-06-16 02:55:42) to be greater than last write (2025-06-16 03:03:10)
+ sleep 10
+ [[ 1750042542 -gt 1750042990 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ABq7dZBtaK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.TPipStWcKw
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ABq7dZBtaK
+++ cat /tmp/tmp.TPipStWcKw
+++ rm /tmp/tmp.ABq7dZBtaK /tmp/tmp.TPipStWcKw
+++ return 0
++ echo 1750043068
+ last_chunk=1750043068
+ retries=7
++ format_date 1750043068
++ local timestamp=1750043068
+++ TZ=UTC
+++ /usr/bin/date -d@1750043068 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 03:04:28
++ format_date 1750042990
++ local timestamp=1750042990
+++ TZ=UTC
+++ /usr/bin/date -d@1750042990 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 03:03:10
+ log 'Waiting for last oplog chunk (2025-06-16 03:04:28) to be greater than last write (2025-06-16 03:03:10)'
+ set +o xtrace
[2025-06-16T03:04:41+0000] Waiting for last oplog chunk (2025-06-16 03:04:28) to be greater than last write (2025-06-16 03:03:10)
+ sleep 10
+ [[ 1750043068 -gt 1750042990 ]]
+++ get_latest_oplog_chunk_ts some-name
+++ local cluster=some-name
++++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.UboPLy3aNI
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.Gpt9XJr9uI
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.UboPLy3aNI
++++ cat /tmp/tmp.Gpt9XJr9uI
++++ rm /tmp/tmp.UboPLy3aNI /tmp/tmp.Gpt9XJr9uI
++++ return 0
+++ echo 1750043068
++ format_date 1750043068
++ local timestamp=1750043068
+++ TZ=UTC
+++ /usr/bin/date -d@1750043068 '+%Y-%m-%d %H:%M:%S'
++ echo 2025-06-16 03:04:28
+ local 'target_time=2025-06-16 03:04:28'
+ log 'dropping test collection'
+ set +o xtrace
[2025-06-16T03:04:53+0000] dropping test collection
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QNmWSXa6Ru
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2kiVyNAdN9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QNmWSXa6Ru
++ cat /tmp/tmp.2kiVyNAdN9
++ rm /tmp/tmp.QNmWSXa6Ru /tmp/tmp.2kiVyNAdN9
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DI5N7YEZ9t
++ mktemp
+ local LAST_ERR=/tmp/tmp.xcJMD0Awyw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DI5N7YEZ9t
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-fs-25161.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("597242df-6fb6-4610-890f-6b1e7b98bd5b") }
Percona Server for MongoDB server version: v8.0.8-3
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.xcJMD0Awyw
+ rm /tmp/tmp.DI5N7YEZ9t /tmp/tmp.xcJMD0Awyw
+ return 0
+ log 'checking pitr... backup: backup-nfs-physical-pitr target: 2025-06-16 03:04:28'
+ set +o xtrace
[2025-06-16T03:04:56+0000] checking pitr... backup: backup-nfs-physical-pitr target: 2025-06-16 03:04:28
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/conf/pitr.yml
+ yq eval '.metadata.name = "restore-backup-nfs-physical-pitr"'
+ yq eval '.spec.pitr.date = "2025-06-16 03:04:28"'
+ kubectl_bin apply -f -
+ yq eval '.spec.backupName = "backup-nfs-physical-pitr"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.fSxUxB1lZ5
++ mktemp
+ local LAST_ERR=/tmp/tmp.jTD0MDkH59
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fSxUxB1lZ5
perconaservermongodbrestore.psmdb.percona.com/restore-backup-nfs-physical-pitr created
+ cat /tmp/tmp.jTD0MDkH59
+ rm /tmp/tmp.fSxUxB1lZ5 /tmp/tmp.jTD0MDkH59
+ return 0
+ wait_restore backup-nfs-physical-pitr some-name
+ local backup_name=backup-nfs-physical-pitr
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-nfs-physical-pitr object to be createdOK
Waiting psmdb-restore/restore-backup-nfs-physical-pitr to reach state "ready" ....OK after 4 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vd4jnf4VLR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1rWpsSwJMy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vd4jnf4VLR
++ cat /tmp/tmp.1rWpsSwJMy
++ rm /tmp/tmp.vd4jnf4VLR /tmp/tmp.1rWpsSwJMy
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UmxrLMZFyN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.e3Yp2JZd3M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UmxrLMZFyN
++ cat /tmp/tmp.e3Yp2JZd3M
++ rm /tmp/tmp.UmxrLMZFyN /tmp/tmp.e3Yp2JZd3M
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4hFlL5NtDK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.w7yYh9sJVh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4hFlL5NtDK
++ cat /tmp/tmp.w7yYh9sJVh
++ rm /tmp/tmp.4hFlL5NtDK /tmp/tmp.w7yYh9sJVh
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C8NMmGY8sU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rFTYQYYCcy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C8NMmGY8sU
++ cat /tmp/tmp.rFTYQYYCcy
++ rm /tmp/tmp.C8NMmGY8sU /tmp/tmp.rFTYQYYCcy
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uVWJ0ElGXy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.h6KCOsTQaU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uVWJ0ElGXy
++ cat /tmp/tmp.h6KCOsTQaU
++ rm /tmp/tmp.uVWJ0ElGXy /tmp/tmp.h6KCOsTQaU
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lZ3QUSGy5U
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o6pkLKlzYI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lZ3QUSGy5U
++ cat /tmp/tmp.o6pkLKlzYI
++ rm /tmp/tmp.lZ3QUSGy5U /tmp/tmp.o6pkLKlzYI
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oNEdtNYxWa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qYU6iRXp2m
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oNEdtNYxWa
++ cat /tmp/tmp.qYU6iRXp2m
++ rm /tmp/tmp.oNEdtNYxWa /tmp/tmp.qYU6iRXp2m
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GfzJfTTkgh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IXyLc17EdU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GfzJfTTkgh
++ cat /tmp/tmp.IXyLc17EdU
++ rm /tmp/tmp.GfzJfTTkgh /tmp/tmp.IXyLc17EdU
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fG7NDYIfIK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8K7LMlzuJ0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fG7NDYIfIK
++ cat /tmp/tmp.8K7LMlzuJ0
++ rm /tmp/tmp.fG7NDYIfIK /tmp/tmp.8K7LMlzuJ0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UF6htMIbL6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RMpGPG2LSY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UF6htMIbL6
++ cat /tmp/tmp.RMpGPG2LSY
++ rm /tmp/tmp.UF6htMIbL6 /tmp/tmp.RMpGPG2LSY
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1i4AacusYo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ALq3SfNgcl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1i4AacusYo
++ cat /tmp/tmp.ALq3SfNgcl
++ rm /tmp/tmp.1i4AacusYo /tmp/tmp.ALq3SfNgcl
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lYXZ4FPV5H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.duFWqa5F0p
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lYXZ4FPV5H
++ cat /tmp/tmp.duFWqa5F0p
++ rm /tmp/tmp.lYXZ4FPV5H /tmp/tmp.duFWqa5F0p
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EV5G2wfIyT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Nd2v6tyKWd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EV5G2wfIyT
++ cat /tmp/tmp.Nd2v6tyKWd
++ rm /tmp/tmp.EV5G2wfIyT /tmp/tmp.Nd2v6tyKWd
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AZzPVElz0v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6RLExCkuqP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AZzPVElz0v
++ cat /tmp/tmp.6RLExCkuqP
++ rm /tmp/tmp.AZzPVElz0v /tmp/tmp.6RLExCkuqP
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-fs-25161 -5th .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ local postfix=-5th
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-06-16T03:12:11+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-fs-25161 mongodb .svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-fs-25161
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kA3hMMtzNp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SdkC3AzCQU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kA3hMMtzNp
++ cat /tmp/tmp.SdkC3AzCQU
++ rm /tmp/tmp.kA3hMMtzNp /tmp/tmp.SdkC3AzCQU
++ return 0
+ local client_container=psmdb-client-66f577db5f-rbmr7
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.demand-backup-fs-25161 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Xt2tVms92b
++ mktemp
+ local LAST_ERR=/tmp/tmp.Yvq0BaUd56
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-rbmr7 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-fs-25161.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Xt2tVms92b
+ cat /tmp/tmp.Yvq0BaUd56
+ rm /tmp/tmp.Xt2tVms92b /tmp/tmp.Yvq0BaUd56
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/e2e-tests/demand-backup-fs/compare/find-5th.json /tmp/tmp.JnbyHFcDhi/find-5th
+ destroy demand-backup-fs-25161
+ local namespace=demand-backup-fs-25161
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q4ZeiWsQXK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2xkuMcyI2A
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.q4ZeiWsQXK
++ cat /tmp/tmp.2xkuMcyI2A
++ rm /tmp/tmp.q4ZeiWsQXK /tmp/tmp.2xkuMcyI2A
++ return 0
+ '[' 4 '!=' 0 ']'
+ kubectl_bin get psmdb-backup
++ mktemp
+ local LAST_OUT=/tmp/tmp.VfaTK2FBx7
++ mktemp
+ local LAST_ERR=/tmp/tmp.bYMdXPQAcu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get psmdb-backup
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VfaTK2FBx7
NAME                       CLUSTER     STORAGE   DESTINATION                     TYPE       SIZE       STATUS   COMPLETED   AGE
backup-nfs-logical         some-name   nfs       /mnt/nfs/2025-06-16T02:52:41Z   logical    38.31KB    ready    19m         19m
backup-nfs-logical-pitr    some-name   nfs       /mnt/nfs/2025-06-16T02:53:38Z   logical    43.44KB    ready    18m         18m
backup-nfs-physical        some-name   nfs       /mnt/nfs/2025-06-16T02:56:08Z   physical   1.32MB     ready    16m         16m
backup-nfs-physical-pitr   some-name   nfs       /mnt/nfs/2025-06-16T03:03:08Z   physical   868.65KB   ready    9m1s        9m8s
+ cat /tmp/tmp.bYMdXPQAcu
+ rm /tmp/tmp.VfaTK2FBx7 /tmp/tmp.bYMdXPQAcu
+ return 0
+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.54pgZGmdW7
++ mktemp
+ local LAST_ERR=/tmp/tmp.2WoHo1R1iL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.54pgZGmdW7
perconaservermongodbbackup.psmdb.percona.com "backup-nfs-logical" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-nfs-logical-pitr" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-nfs-physical" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-nfs-physical-pitr" deleted
+ cat /tmp/tmp.2WoHo1R1iL
+ rm /tmp/tmp.54pgZGmdW7 /tmp/tmp.2WoHo1R1iL
+ return 0
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.yEolnDnM9o
++ mktemp
+ local LAST_ERR=/tmp/tmp.PJJrEcjEXj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yEolnDnM9o
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.PJJrEcjEXj
+ rm /tmp/tmp.yEolnDnM9o /tmp/tmp.PJJrEcjEXj
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.DJ2Z39A96N
++ mktemp
+ local LAST_ERR=/tmp/tmp.4vrIAgS5aG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DJ2Z39A96N
+ cat /tmp/tmp.4vrIAgS5aG
+ rm /tmp/tmp.DJ2Z39A96N /tmp/tmp.4vrIAgS5aG
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.NezaVXn8RW
++ mktemp
+ local LAST_ERR=/tmp/tmp.Gw8n5FEN2I
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NezaVXn8RW
+ cat /tmp/tmp.Gw8n5FEN2I
+ rm /tmp/tmp.NezaVXn8RW /tmp/tmp.Gw8n5FEN2I
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n demand-backup-fs-25161 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.hYrMTb8YAq
++ mktemp
+ local LAST_ERR=/tmp/tmp.5xdehJHZkw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hYrMTb8YAq
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met
+ cat /tmp/tmp.5xdehJHZkw
+ rm /tmp/tmp.hYrMTb8YAq /tmp/tmp.5xdehJHZkw
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.Im8hNFTYk9
++ mktemp
+ local LAST_ERR=/tmp/tmp.Q6sfbXI2Hx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1969/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Im8hNFTYk9
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.Q6sfbXI2Hx
+ rm /tmp/tmp.Im8hNFTYk9 /tmp/tmp.Q6sfbXI2Hx
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.42PGsMDKtr
++ mktemp
+ local LAST_ERR=/tmp/tmp.sohFrDpvf1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.42PGsMDKtr
+ cat /tmp/tmp.sohFrDpvf1
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.42PGsMDKtr
+ cat /tmp/tmp.sohFrDpvf1
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.42PGsMDKtr
+ cat /tmp/tmp.sohFrDpvf1
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.42PGsMDKtr
+ cat /tmp/tmp.sohFrDpvf1
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.42PGsMDKtr /tmp/tmp.sohFrDpvf1
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-fs-25161
+ rm -rf /tmp/tmp.JnbyHFcDhi
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.Gu7EOPP1GZ
+ desc 'test passed'
+ set +o xtrace
+ local LAST_OUT=/tmp/tmp.OTGJzQuM2X
++ mktemp


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

++ mktemp
+ local LAST_ERR=/tmp/tmp.xV30CiQTNJ
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.9ZQeY2oL5R
+ local exit_status=0
++ seq 0 2
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-fs-25161
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator