Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/logs/demand-backup-incremental-sharded.log
grep: warning: stray \ before -
Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1
+ create_infra demand-backup-incremental-sharded-19753
+ local ns=demand-backup-incremental-sharded-19753
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.1QCulyxRKw
++ mktemp
+ local LAST_ERR=/tmp/tmp.0QzSg1H5JQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1QCulyxRKw
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.0QzSg1H5JQ
+ rm /tmp/tmp.1QCulyxRKw /tmp/tmp.0QzSg1H5JQ
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-19253 backup-aws-s3-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-19253 backup-azure-blob-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-19253 backup-gcp-cs-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "backup-gcp-cs-sharded" not found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-19253 backup-minio-not-base-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-not-base-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-19253 backup-minio-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded patched
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.HyFt7VFUic
++ mktemp
+ local LAST_ERR=/tmp/tmp.VuLT1Q3XxG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HyFt7VFUic
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.VuLT1Q3XxG
+ rm /tmp/tmp.HyFt7VFUic /tmp/tmp.VuLT1Q3XxG
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.V7e5uQT3J5
++ mktemp
+ local LAST_ERR=/tmp/tmp.t6dVrs80EG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V7e5uQT3J5
+ cat /tmp/tmp.t6dVrs80EG
+ rm /tmp/tmp.V7e5uQT3J5 /tmp/tmp.t6dVrs80EG
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.vie07bYejh
++ mktemp
+ local LAST_ERR=/tmp/tmp.qZxM3hGAdt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vie07bYejh
+ cat /tmp/tmp.qZxM3hGAdt
+ rm /tmp/tmp.vie07bYejh /tmp/tmp.qZxM3hGAdt
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.wdofiWH2qP
++ mktemp
+ local LAST_ERR=/tmp/tmp.zybuh2o60k
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wdofiWH2qP
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.zybuh2o60k
+ rm /tmp/tmp.wdofiWH2qP /tmp/tmp.zybuh2o60k
+ return 0
+ check_crd_for_deletion PR-2083-3a393d8f
+ local git_tag=PR-2083-3a393d8f
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2083-3a393d8f/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kj4mYVHBv9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xTxH4EK4PX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.kj4mYVHBv9
++ cat /tmp/tmp.xTxH4EK4PX
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.kj4mYVHBv9
++ cat /tmp/tmp.xTxH4EK4PX
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.kj4mYVHBv9
++ cat /tmp/tmp.xTxH4EK4PX
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.kj4mYVHBv9
++ cat /tmp/tmp.xTxH4EK4PX
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.kj4mYVHBv9 /tmp/tmp.xTxH4EK4PX
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace
++ mktemp


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.ByFOeZvEdX
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_OUT=/tmp/tmp.qhR3sknBW6
++ mktemp
+ local LAST_ERR=/tmp/tmp.eELWf2f6jz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.zxcKfKl6le
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ByFOeZvEdX
+ cat /tmp/tmp.eELWf2f6jz
+ rm /tmp/tmp.ByFOeZvEdX /tmp/tmp.eELWf2f6jz
+ return 0
namespace "demand-backup-incremental-sharded-19253" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qhR3sknBW6
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.zxcKfKl6le
+ rm /tmp/tmp.qhR3sknBW6 /tmp/tmp.zxcKfKl6le
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.9zQMGmJ5vn
++ mktemp
+ local LAST_ERR=/tmp/tmp.bj8OMeUswQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9zQMGmJ5vn
+ cat /tmp/tmp.bj8OMeUswQ
+ rm /tmp/tmp.9zQMGmJ5vn /tmp/tmp.bj8OMeUswQ
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.78IVTtQltj
++ mktemp
+ local LAST_ERR=/tmp/tmp.QsTwBB4BHt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.78IVTtQltj
namespace/psmdb-operator created
+ cat /tmp/tmp.QsTwBB4BHt
+ rm /tmp/tmp.78IVTtQltj /tmp/tmp.QsTwBB4BHt
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YpKPX15YRw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xHQo6c2m3q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YpKPX15YRw
++ cat /tmp/tmp.xHQo6c2m3q
++ rm /tmp/tmp.YpKPX15YRw /tmp/tmp.xHQo6c2m3q
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2083-3a393d8f-3-cluster8 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.RjvX8amA21
++ mktemp
+ local LAST_ERR=/tmp/tmp.iIO5O7FeGQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2083-3a393d8f-3-cluster8 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RjvX8amA21
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2083-3a393d8f-3-cluster8" modified.
+ cat /tmp/tmp.iIO5O7FeGQ
+ rm /tmp/tmp.RjvX8amA21 /tmp/tmp.iIO5O7FeGQ
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2083-3a393d8f'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2083-3a393d8f
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.xwDBnlSkOi
++ mktemp
+ local LAST_ERR=/tmp/tmp.GYRJS19Z3q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xwDBnlSkOi
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.GYRJS19Z3q
+ rm /tmp/tmp.xwDBnlSkOi /tmp/tmp.GYRJS19Z3q
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.9Lk1dUWnwt
++ mktemp
+ local LAST_ERR=/tmp/tmp.aG9hmity5M
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9Lk1dUWnwt
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.aG9hmity5M
+ rm /tmp/tmp.9Lk1dUWnwt /tmp/tmp.aG9hmity5M
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2083-3a393d8f") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.RSrpvglFJE
++ mktemp
+ local LAST_ERR=/tmp/tmp.cY8M31oBu6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RSrpvglFJE
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.cY8M31oBu6
+ rm /tmp/tmp.RSrpvglFJE /tmp/tmp.cY8M31oBu6
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D8pd2MavQF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bcbQTGuz6c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D8pd2MavQF
++ cat /tmp/tmp.bcbQTGuz6c
++ rm /tmp/tmp.D8pd2MavQF /tmp/tmp.bcbQTGuz6c
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-5879655bb8-pm2lc
+ local pod=percona-server-mongodb-operator-5879655bb8-pm2lc
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-5879655bb8-pm2lc to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vLQVBQ9rzQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rXdbE51GvJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vLQVBQ9rzQ
++ cat /tmp/tmp.rXdbE51GvJ
++ rm /tmp/tmp.vLQVBQ9rzQ /tmp/tmp.rXdbE51GvJ
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-5879655bb8-pm2lc
++ mktemp
+ local LAST_OUT=/tmp/tmp.fgvBbz84NW
++ mktemp
+ local LAST_ERR=/tmp/tmp.IpyAhafjew
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-5879655bb8-pm2lc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fgvBbz84NW
+ cat /tmp/tmp.IpyAhafjew
+ rm /tmp/tmp.fgvBbz84NW /tmp/tmp.IpyAhafjew
+ return 0
2025-10-09T18:56:19.291Z	INFO	setup	Manager starting up	{"gitCommit": "3a393d8ff1e5ff2eba0fbfc49de1c98056cd62c6", "gitBranch": "PR-2083-3a393d8f", "buildTime": "", "goVersion": "go1.25.2", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-incremental-sharded-19753
+ local namespace=demand-backup-incremental-sharded-19753
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-incremental-sharded-19753'
+ xargs kubectl delete ns
+ awk '{print$1}'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-incremental-sharded-19753
-----------------------------------------------------------------------------------

++ mktemp
+ kubectl_bin delete namespace demand-backup-incremental-sharded-19753 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.KWpBPlOk8g
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_OUT=/tmp/tmp.Bs8mhW2pw8
++ mktemp
+ local LAST_ERR=/tmp/tmp.b2h84BcgtD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.tQot5Rgabl
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-incremental-sharded-19753 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KWpBPlOk8g
+ cat /tmp/tmp.b2h84BcgtD
+ rm /tmp/tmp.KWpBPlOk8g /tmp/tmp.b2h84BcgtD
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Bs8mhW2pw8
+ cat /tmp/tmp.tQot5Rgabl
+ rm /tmp/tmp.Bs8mhW2pw8 /tmp/tmp.tQot5Rgabl
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-incremental-sharded-19753
++ mktemp
+ local LAST_OUT=/tmp/tmp.LSIr5e3MHL
++ mktemp
+ local LAST_ERR=/tmp/tmp.lnDgrvMUTv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-incremental-sharded-19753
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LSIr5e3MHL
+ cat /tmp/tmp.lnDgrvMUTv
+ rm /tmp/tmp.LSIr5e3MHL /tmp/tmp.lnDgrvMUTv
+ return 0
+ desc 'create namespace demand-backup-incremental-sharded-19753'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-incremental-sharded-19753
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-incremental-sharded-19753
++ mktemp
+ local LAST_OUT=/tmp/tmp.Rca8zgo4Dm
++ mktemp
+ local LAST_ERR=/tmp/tmp.e2kJxAxvnD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-incremental-sharded-19753
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Rca8zgo4Dm
namespace/demand-backup-incremental-sharded-19753 created
+ cat /tmp/tmp.e2kJxAxvnD
+ rm /tmp/tmp.Rca8zgo4Dm /tmp/tmp.e2kJxAxvnD
+ return 0
+ set_kube_ctx demand-backup-incremental-sharded-19753
+ local namespace=demand-backup-incremental-sharded-19753
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7BIKyV3MLH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q5MPphl8YM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7BIKyV3MLH
++ cat /tmp/tmp.Q5MPphl8YM
++ rm /tmp/tmp.7BIKyV3MLH /tmp/tmp.Q5MPphl8YM
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2083-3a393d8f-3-cluster8 --namespace=demand-backup-incremental-sharded-19753
++ mktemp
+ local LAST_OUT=/tmp/tmp.vT0FOU6dU3
++ mktemp
+ local LAST_ERR=/tmp/tmp.CXHIdiNgq9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2083-3a393d8f-3-cluster8 --namespace=demand-backup-incremental-sharded-19753
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vT0FOU6dU3
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2083-3a393d8f-3-cluster8" modified.
+ cat /tmp/tmp.CXHIdiNgq9
+ rm /tmp/tmp.vT0FOU6dU3 /tmp/tmp.CXHIdiNgq9
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Thu Oct  9 18:56:55 2025
NAMESPACE: demand-backup-incremental-sharded-19753
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-incremental-sharded-19753.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-incremental-sharded-19753 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-incremental-sharded-19753

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-incremental-sharded-19753 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-incremental-sharded-19753 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iAxAnQXYDf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4MBcMZ0X6F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iAxAnQXYDf
++ cat /tmp/tmp.4MBcMZ0X6F
++ rm /tmp/tmp.iAxAnQXYDf /tmp/tmp.4MBcMZ0X6F
++ return 0
+ MINIO_POD=minio-service-86dfccd949-w7nqz
+ wait_pod minio-service-86dfccd949-w7nqz
+ local pod=minio-service-86dfccd949-w7nqz
+ set +o xtrace
waiting for pod/minio-service-86dfccd949-w7nqz to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-incremental-sharded-19753.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.vCC2xfEo0W
++ mktemp
+ local LAST_ERR=/tmp/tmp.6BEuTrn9AS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-incremental-sharded-19753.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vCC2xfEo0W
service/minio-service created
+ cat /tmp/tmp.6BEuTrn9AS
+ rm /tmp/tmp.vCC2xfEo0W /tmp/tmp.6BEuTrn9AS
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.wkM3myy8gC
++ mktemp
+ local LAST_ERR=/tmp/tmp.sudFBOc08a
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wkM3myy8gC
make_bucket: operator-testing
pod "aws-cli" deleted from demand-backup-incremental-sharded-19753 namespace
+ cat /tmp/tmp.sudFBOc08a
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-incremental-sharded-19753
+ rm /tmp/tmp.wkM3myy8gC /tmp/tmp.sudFBOc08a
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.8ORQ8732Jg
++ mktemp
+ local LAST_ERR=/tmp/tmp.mcqhECelFB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8ORQ8732Jg
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.mcqhECelFB
+ rm /tmp/tmp.8ORQ8732Jg /tmp/tmp.mcqhECelFB
+ return 0
+ desc 'Testing on sharded cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Testing on sharded cluster
-----------------------------------------------------------------------------------

+ log 'Creating PSMDB cluster'
+ set +o xtrace
[2025-10-09T18:57:35+0000] Creating PSMDB cluster
+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.qAzc0XDvHc
++ mktemp
+ local LAST_ERR=/tmp/tmp.IrC6ktdbCd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qAzc0XDvHc
secret/some-users created
+ cat /tmp/tmp.IrC6ktdbCd
+ rm /tmp/tmp.qAzc0XDvHc /tmp/tmp.IrC6ktdbCd
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2083-3a393d8f"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.7odp9h1LYm
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dq4rXLYHeE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7odp9h1LYm
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.Dq4rXLYHeE
+ rm /tmp/tmp.7odp9h1LYm /tmp/tmp.Dq4rXLYHeE
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.8y8E06sci3
++ mktemp
+ local LAST_ERR=/tmp/tmp.89f3DB59xh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8y8E06sci3
deployment.apps/psmdb-client created
+ cat /tmp/tmp.89f3DB59xh
+ rm /tmp/tmp.8y8E06sci3 /tmp/tmp.89f3DB59xh
+ return 0
+ log 'check if all pods started'
+ set +o xtrace
[2025-10-09T18:57:41+0000] check if all pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aZ8O2eiRwh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Fu4UZ59NcD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aZ8O2eiRwh
++ cat /tmp/tmp.Fu4UZ59NcD
++ rm /tmp/tmp.aZ8O2eiRwh /tmp/tmp.Fu4UZ59NcD
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tGidhorngo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6nn8kK37on
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tGidhorngo
++ cat /tmp/tmp.6nn8kK37on
++ rm /tmp/tmp.tGidhorngo /tmp/tmp.6nn8kK37on
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XRnwKIeSkq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1cLUN8Xgay
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XRnwKIeSkq
++ cat /tmp/tmp.1cLUN8Xgay
++ rm /tmp/tmp.XRnwKIeSkq /tmp/tmp.1cLUN8Xgay
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..............................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Qi4sdTZDAY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fQh0GehLLx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Qi4sdTZDAY
++ cat /tmp/tmp.fQh0GehLLx
++ rm /tmp/tmp.Qi4sdTZDAY /tmp/tmp.fQh0GehLLx
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KYvYxSoG5s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.68hUSuzQGH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KYvYxSoG5s
++ cat /tmp/tmp.68hUSuzQGH
++ rm /tmp/tmp.KYvYxSoG5s /tmp/tmp.68hUSuzQGH
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EVe0S0for9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d2A4x6TA7G
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EVe0S0for9
++ cat /tmp/tmp.d2A4x6TA7G
++ rm /tmp/tmp.EVe0S0for9 /tmp/tmp.d2A4x6TA7G
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jbkzSNe0AK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.81vsRJ9qb4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jbkzSNe0AK
++ cat /tmp/tmp.81vsRJ9qb4
++ rm /tmp/tmp.jbkzSNe0AK /tmp/tmp.81vsRJ9qb4
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LITwOrTbH0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QOVNOF2CDl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LITwOrTbH0
++ cat /tmp/tmp.QOVNOF2CDl
++ rm /tmp/tmp.LITwOrTbH0 /tmp/tmp.QOVNOF2CDl
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KIuCBCocoB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UWjfyn9F2F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KIuCBCocoB
++ cat /tmp/tmp.UWjfyn9F2F
++ rm /tmp/tmp.KIuCBCocoB /tmp/tmp.UWjfyn9F2F
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1ezSYSlrlJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eirbH4XWB8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1ezSYSlrlJ
++ cat /tmp/tmp.eirbH4XWB8
++ rm /tmp/tmp.1ezSYSlrlJ /tmp/tmp.eirbH4XWB8
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
++ kubectl_bin get svc some-name-mongos '-o=jsonpath={.status}'
++ jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0] | if .ip then .ip else .hostname end'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.v1u2pL3DAi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bXqgmTqtdr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get svc some-name-mongos '-o=jsonpath={.status}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.v1u2pL3DAi
++ cat /tmp/tmp.bXqgmTqtdr
++ rm /tmp/tmp.v1u2pL3DAi /tmp/tmp.bXqgmTqtdr
++ return 0
+ lbEndpoint=34.58.224.177
+ '[' -z 34.58.224.177 ']'
+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-19753
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1gQ2oVH1AW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pFUSUWJr94
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1gQ2oVH1AW
++ cat /tmp/tmp.pFUSUWJr94
++ rm /tmp/tmp.1gQ2oVH1AW /tmp/tmp.pFUSUWJr94
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.MgiqixyLux
++ mktemp
+ local LAST_ERR=/tmp/tmp.7QEu8PxP1t
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MgiqixyLux
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("58034627-3bd2-4970-91f6-f05d12f0bbfe") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.7QEu8PxP1t
+ rm /tmp/tmp.MgiqixyLux /tmp/tmp.7QEu8PxP1t
+ return 0
+ sleep 1
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G7a5XVKae7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qDiSIatIZo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.G7a5XVKae7
++ cat /tmp/tmp.qDiSIatIZo
++ rm /tmp/tmp.G7a5XVKae7 /tmp/tmp.qDiSIatIZo
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.o6SIoRMvF4
++ mktemp
+ local LAST_ERR=/tmp/tmp.zHRDwtg1FQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.o6SIoRMvF4
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b85f870f-177d-4c78-b08e-1ffc57e7d2bb") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.zHRDwtg1FQ
+ rm /tmp/tmp.o6SIoRMvF4 /tmp/tmp.zHRDwtg1FQ
+ return 0
+ sleep 5
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QGLNfNhsZH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DbVyQkhIqT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QGLNfNhsZH
++ cat /tmp/tmp.DbVyQkhIqT
++ rm /tmp/tmp.QGLNfNhsZH /tmp/tmp.DbVyQkhIqT
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mIH4bH0ZVW
++ mktemp
+ local LAST_ERR=/tmp/tmp.UXkPRU3KSO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mIH4bH0ZVW
+ cat /tmp/tmp.UXkPRU3KSO
+ rm /tmp/tmp.mIH4bH0ZVW /tmp/tmp.UXkPRU3KSO
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.3UJ4bnYUHf/find-sharded
+ log 'waiting 60 seconds for stable timestamp in wiredtiger'
+ set +o xtrace
[2025-10-09T19:01:03+0000] waiting 60 seconds for stable timestamp in wiredtiger
+ sleep 80
+ log 'running backups'
+ set +o xtrace
[2025-10-09T19:02:23+0000] running backups
+ '[' -z '' ']'
+ backup_name_aws=backup-aws-s3-sharded
+ backup_name_gcp=backup-gcp-cs-sharded
+ backup_name_azure=backup-azure-blob-sharded
+ run_backup aws-s3 backup-aws-s3-sharded
+ local storage=aws-s3
+ local backup_name=backup-aws-s3-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: backup-aws-s3-sharded/'
+ /usr/sbin/sed -e 's/storageName:/storageName: aws-s3/'
+ yq '.spec.type="incremental-base"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ucqB85Wkmg
++ mktemp
+ local LAST_ERR=/tmp/tmp.Snn9P58vdF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ucqB85Wkmg
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded created
+ cat /tmp/tmp.Snn9P58vdF
+ rm /tmp/tmp.ucqB85Wkmg /tmp/tmp.Snn9P58vdF
+ return 0
+ run_backup gcp-cs backup-gcp-cs-sharded
+ local storage=gcp-cs
+ local backup_name=backup-gcp-cs-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: backup-gcp-cs-sharded/'
+ /usr/sbin/sed -e 's/storageName:/storageName: gcp-cs/'
+ yq '.spec.type="incremental-base"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.MOg470D7dH
++ mktemp
+ local LAST_ERR=/tmp/tmp.GujMt0MHez
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MOg470D7dH
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded created
+ cat /tmp/tmp.GujMt0MHez
+ rm /tmp/tmp.MOg470D7dH /tmp/tmp.GujMt0MHez
+ return 0
+ run_backup azure-blob backup-azure-blob-sharded
+ local storage=azure-blob
+ local backup_name=backup-azure-blob-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: backup-azure-blob-sharded/'
+ /usr/sbin/sed -e 's/storageName:/storageName: azure-blob/'
+ yq '.spec.type="incremental-base"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fSfsTaBfrG
++ mktemp
+ local LAST_ERR=/tmp/tmp.JVYC9cB3xR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fSfsTaBfrG
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded created
+ cat /tmp/tmp.JVYC9cB3xR
+ rm /tmp/tmp.fSfsTaBfrG /tmp/tmp.JVYC9cB3xR
+ return 0
+ wait_backup backup-aws-s3-sharded
+ local backup_name=backup-aws-s3-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3-sharded to reach ready state.....................OK
+ check_backup_in_storage backup-aws-s3-sharded s3 rs0
+ local backup=backup-aws-s3-sharded
+ local storage_type=s3
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=s3.amazonaws.com
++ get_backup_dest backup-aws-s3-sharded
++ local backup_name=backup-aws-s3-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.mfsGodSGul
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pxZEZ8a87W
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mfsGodSGul
++ cat /tmp/tmp.pxZEZ8a87W
++ rm /tmp/tmp.mfsGodSGul /tmp/tmp.pxZEZ8a87W
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:02:26Z
+ [[ s3 == \m\i\n\i\o ]]
+ local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:02:26Z/rs0/filelist.pbm
+ log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:02:26Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-10-09T19:03:09+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:02:26Z/rs0/filelist.pbm exists
+ curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:02:26Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 10067    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
x-amz-id-2: Lv9XHsUmHj527VWZf2VJnCFBK3NR/b4GjJWuwkvVQNyfrpg/Bwm/jsvkCzr3AEG8uEqZWb+jiAKcZtUIJ0hBJ3BiEb/Da6LiZ0ydF4dKZd8=
x-amz-request-id: 2R2QWTB0EXG2PRCV
Date: Thu, 09 Oct 2025 19:03:10 GMT
Last-Modified: Thu, 09 Oct 2025 19:02:49 GMT
x-amz-expiration: expiry-date="Sat, 11 Oct 2025 00:00:00 GMT", rule-id="1 Days Cleanup"
ETag: "be25559fde85bea6613c36b33e9e18cd"
x-amz-server-side-encryption: AES256
Accept-Ranges: bytes
Content-Type: application/octet-stream
Content-Length: 10067
Server: AmazonS3

+ check_backup_in_storage backup-aws-s3-sharded s3 cfg
+ local backup=backup-aws-s3-sharded
+ local storage_type=s3
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=s3.amazonaws.com
++ get_backup_dest backup-aws-s3-sharded
++ local backup_name=backup-aws-s3-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MqZFCSUNW0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6DYsu96WQm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MqZFCSUNW0
++ cat /tmp/tmp.6DYsu96WQm
++ rm /tmp/tmp.MqZFCSUNW0 /tmp/tmp.6DYsu96WQm
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:02:26Z
+ [[ s3 == \m\i\n\i\o ]]
+ local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:02:26Z/cfg/filelist.pbm
+ log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:02:26Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-10-09T19:03:10+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:02:26Z/cfg/filelist.pbm exists
+ curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:02:26Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 18179    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 18179    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
x-amz-id-2: EDElAC5cAnJ7AihSIxoqCenwO/YZ7rbXXaBS9tNVudiJoMDhTBgLPD7YIkFxhRBhpOSih9mRgyI=
x-amz-request-id: K8YZZR8E2XZ6PNMM
Date: Thu, 09 Oct 2025 19:03:11 GMT
Last-Modified: Thu, 09 Oct 2025 19:03:00 GMT
x-amz-expiration: expiry-date="Sat, 11 Oct 2025 00:00:00 GMT", rule-id="1 Days Cleanup"
ETag: "8fc2362f80a963000b2ac6385325da5a"
x-amz-server-side-encryption: AES256
Accept-Ranges: bytes
Content-Type: application/octet-stream
Content-Length: 18179
Server: AmazonS3

+ wait_backup backup-gcp-cs-sharded
+ local backup_name=backup-gcp-cs-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sharded to reach ready state................OK
+ check_backup_in_storage backup-gcp-cs-sharded gcs rs0
+ local backup=backup-gcp-cs-sharded
+ local storage_type=gcs
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sharded
++ local backup_name=backup-gcp-cs-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.uu2rkcDFun
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dsVR7WFz8q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uu2rkcDFun
++ cat /tmp/tmp.dsVR7WFz8q
++ rm /tmp/tmp.uu2rkcDFun /tmp/tmp.dsVR7WFz8q
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:09Z
+ [[ gcs == \m\i\n\i\o ]]
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:09Z/rs0/filelist.pbm
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:09Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-10-09T19:03:41+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:09Z/rs0/filelist.pbm exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:09Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 12759    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: AAwnv3LzW-r4Of3ScRxNdST_RWtykKLVyCKnDSZ1hMcEvBC8gspNGgtoP20na9kRhyAcLlbL
expires: Thu, 09 Oct 2025 20:03:41 GMT
date: Thu, 09 Oct 2025 19:03:41 GMT
cache-control: public, max-age=3600
last-modified: Thu, 09 Oct 2025 19:03:35 GMT
etag: "875a11d76f986c3f4a90973fc33ff36c"
x-goog-generation: 1760036615961154
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 12759
x-goog-hash: crc32c=ivrnGw==
x-goog-hash: md5=h1oR12+YbD9KkJc/wz/zbA==
x-goog-expiration: Fri, 10 Oct 2025 19:03:35 GMT
x-goog-storage-class: REGIONAL
accept-ranges: bytes
content-length: 12759
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ check_backup_in_storage backup-gcp-cs-sharded gcs cfg
+ local backup=backup-gcp-cs-sharded
+ local storage_type=gcs
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sharded
++ local backup_name=backup-gcp-cs-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.R2gwclKZX3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9dXX5aA3SP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.R2gwclKZX3
++ cat /tmp/tmp.9dXX5aA3SP
++ rm /tmp/tmp.R2gwclKZX3 /tmp/tmp.9dXX5aA3SP
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:09Z
+ [[ gcs == \m\i\n\i\o ]]
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:09Z/cfg/filelist.pbm
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:09Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-10-09T19:03:42+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:09Z/cfg/filelist.pbm exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:09Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 18299    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: AAwnv3KsYIUAdfm3TIltKI13M3JeQWJP4qWi5ZQoo7LzWlxI-Ics1OqYo8is0H0GLc6wzKuk
expires: Thu, 09 Oct 2025 20:03:42 GMT
date: Thu, 09 Oct 2025 19:03:42 GMT
cache-control: public, max-age=3600
last-modified: Thu, 09 Oct 2025 19:03:34 GMT
etag: "2abc0ca3b3908b25121efaa0d476620a"
x-goog-generation: 1760036614827149
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 18299
x-goog-hash: crc32c=ahZgPA==
x-goog-hash: md5=KrwMo7OQiyUSHvqg1HZiCg==
x-goog-expiration: Fri, 10 Oct 2025 19:03:34 GMT
x-goog-storage-class: REGIONAL
accept-ranges: bytes
content-length: 18299
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ wait_backup backup-azure-blob-sharded
+ local backup_name=backup-azure-blob-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob-sharded to reach ready state.................OK
+ check_backup_in_storage backup-azure-blob-sharded azure rs0
+ local backup=backup-azure-blob-sharded
+ local storage_type=azure
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=engk8soperators.blob.core.windows.net
++ get_backup_dest backup-azure-blob-sharded
++ local backup_name=backup-azure-blob-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
+++ mktemp
++ sed -e 's/.json$//'
++ local LAST_OUT=/tmp/tmp.2yqkEBfnld
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BImbfdTWwP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2yqkEBfnld
++ cat /tmp/tmp.BImbfdTWwP
++ rm /tmp/tmp.2yqkEBfnld /tmp/tmp.BImbfdTWwP
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:42Z
+ [[ azure == \m\i\n\i\o ]]
+ local url=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:42Z/rs0/filelist.pbm
+ log 'checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:42Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-10-09T19:04:15+0000] checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:42Z/rs0/filelist.pbm exists
+ curl --fail --head https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:42Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 11885    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
Content-Length: 11885
Content-Type: application/octet-stream
Content-MD5: YIWUjnMS2efsrriviJpLMg==
Last-Modified: Thu, 09 Oct 2025 19:04:02 GMT
ETag: 0x8DE07669C4A82A9
Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
x-ms-request-id: 423a7393-601e-0020-484f-39a4c8000000
x-ms-version: 2009-09-19
x-ms-lease-status: unlocked
x-ms-blob-type: BlockBlob
Date: Thu, 09 Oct 2025 19:04:15 GMT

+ check_backup_in_storage backup-azure-blob-sharded azure cfg
+ local backup=backup-azure-blob-sharded
+ local storage_type=azure
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=engk8soperators.blob.core.windows.net
++ get_backup_dest backup-azure-blob-sharded
++ local backup_name=backup-azure-blob-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.QqolQUBhFO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kl192gMg0H
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QqolQUBhFO
++ cat /tmp/tmp.kl192gMg0H
++ rm /tmp/tmp.QqolQUBhFO /tmp/tmp.kl192gMg0H
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:42Z
+ [[ azure == \m\i\n\i\o ]]
+ local url=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:42Z/cfg/filelist.pbm
+ log 'checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:42Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-10-09T19:04:16+0000] checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:42Z/cfg/filelist.pbm exists
+ curl --fail --head https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:42Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 18179    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
Content-Length: 18179
Content-Type: application/octet-stream
Content-MD5: UGBgf7rokI3vCpjdmAEtPQ==
Last-Modified: Thu, 09 Oct 2025 19:04:05 GMT
ETag: 0x8DE07669E0F4422
Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
x-ms-request-id: 81290214-701e-0085-504f-39f2b2000000
x-ms-version: 2009-09-19
x-ms-lease-status: unlocked
x-ms-blob-type: BlockBlob
Date: Thu, 09 Oct 2025 19:04:16 GMT

+ backup_name_minio=backup-minio-sharded
+ run_backup minio backup-minio-sharded
+ local storage=minio
+ local backup_name=backup-minio-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-sharded/'
+ /usr/sbin/sed -e 's/storageName:/storageName: minio/'
+ yq '.spec.type="incremental-base"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.9x5ozINkaW
++ mktemp
+ local LAST_ERR=/tmp/tmp.4YXWnJcgUA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9x5ozINkaW
perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded created
+ cat /tmp/tmp.4YXWnJcgUA
+ rm /tmp/tmp.9x5ozINkaW /tmp/tmp.4YXWnJcgUA
+ return 0
+ wait_backup backup-minio-sharded
+ local backup_name=backup-minio-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-sharded to reach ready state.........OK
+ check_backup_in_storage backup-minio-sharded minio rs0
+ local backup=backup-minio-sharded
+ local storage_type=minio
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=minio-service
++ get_backup_dest backup-minio-sharded
++ local backup_name=backup-minio-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-minio-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.Q6jqQCUkSk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PFzD9UDctH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Q6jqQCUkSk
++ cat /tmp/tmp.PFzD9UDctH
++ rm /tmp/tmp.Q6jqQCUkSk /tmp/tmp.PFzD9UDctH
++ return 0
+ backup_dest=operator-testing/2025-10-09T19:04:19Z
+ [[ minio == \m\i\n\i\o ]]
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-10-09T19:04:19Z/rs0/filelist.pbm
+ grep filelist.pbm
++ mktemp
+ local LAST_OUT=/tmp/tmp.2xtw03jPnW
++ mktemp
+ local LAST_ERR=/tmp/tmp.wyfJ7BEYeB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-10-09T19:04:19Z/rs0/filelist.pbm
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2xtw03jPnW
+ cat /tmp/tmp.wyfJ7BEYeB
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-incremental-sharded-19753
+ rm /tmp/tmp.2xtw03jPnW /tmp/tmp.wyfJ7BEYeB
+ return 0
2025-10-09 19:04:29      13697 filelist.pbm
+ run_mongos 'use myApp\n db.test.insert({ x: 100502 })' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local 'command=use myApp\n db.test.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zOzcswZR3u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PR8ZGPrwrN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zOzcswZR3u
++ cat /tmp/tmp.PR8ZGPrwrN
++ rm /tmp/tmp.zOzcswZR3u /tmp/tmp.PR8ZGPrwrN
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5C6k7mJEfF
++ mktemp
+ local LAST_ERR=/tmp/tmp.E5np4GZ7fd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5C6k7mJEfF
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("63e41eba-34a2-4afe-8ab7-f816d3ef70c0") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.E5np4GZ7fd
+ rm /tmp/tmp.5C6k7mJEfF /tmp/tmp.E5np4GZ7fd
+ return 0
+ sleep 5
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 -not-base-sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local postfix=-not-base-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KHKYpL0VnO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4g4DnAV2cX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KHKYpL0VnO
++ cat /tmp/tmp.4g4DnAV2cX
++ rm /tmp/tmp.KHKYpL0VnO /tmp/tmp.4g4DnAV2cX
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.hfwEaFQZBg
++ mktemp
+ local LAST_ERR=/tmp/tmp.C7MmUcGQWC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hfwEaFQZBg
+ cat /tmp/tmp.C7MmUcGQWC
+ rm /tmp/tmp.hfwEaFQZBg /tmp/tmp.C7MmUcGQWC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/find-not-base-sharded.json /tmp/tmp.3UJ4bnYUHf/find-not-base-sharded
+ backup_name_minio_not_base=backup-minio-not-base-sharded
+ run_backup minio backup-minio-not-base-sharded false
+ local storage=minio
+ local backup_name=backup-minio-not-base-sharded
+ local base=false
+ local backup_type=incremental
+ [[ false == \t\r\u\e ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/sbin/sed -e 's/name:/name: backup-minio-not-base-sharded/'
+ /usr/sbin/sed -e 's/storageName:/storageName: minio/'
+ yq '.spec.type="incremental"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.JWswEyDfk9
++ mktemp
+ local LAST_ERR=/tmp/tmp.SDRDPGOmbv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JWswEyDfk9
perconaservermongodbbackup.psmdb.percona.com/backup-minio-not-base-sharded created
+ cat /tmp/tmp.SDRDPGOmbv
+ rm /tmp/tmp.JWswEyDfk9 /tmp/tmp.SDRDPGOmbv
+ return 0
+ wait_backup backup-minio-not-base-sharded
+ local backup_name=backup-minio-not-base-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-not-base-sharded to reach ready state.........OK
+ check_backup_in_storage backup-minio-not-base-sharded minio rs0
+ local backup=backup-minio-not-base-sharded
+ local storage_type=minio
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=minio-service
++ get_backup_dest backup-minio-not-base-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ local backup_name=backup-minio-not-base-sharded
++ kubectl_bin get psmdb-backup backup-minio-not-base-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.edI26kWQRm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cnrBSpeaPs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio-not-base-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.edI26kWQRm
++ cat /tmp/tmp.cnrBSpeaPs
++ rm /tmp/tmp.edI26kWQRm /tmp/tmp.cnrBSpeaPs
++ return 0
+ backup_dest=operator-testing/2025-10-09T19:04:53Z
+ [[ minio == \m\i\n\i\o ]]
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-10-09T19:04:53Z/rs0/filelist.pbm
+ grep filelist.pbm
++ mktemp
+ local LAST_OUT=/tmp/tmp.2wmbOyc5yt
++ mktemp
+ local LAST_ERR=/tmp/tmp.KCbdK1R1Xh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-10-09T19:04:53Z/rs0/filelist.pbm
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2wmbOyc5yt
+ cat /tmp/tmp.KCbdK1R1Xh
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: Internal error occurred: error attaching to container: failed to load task: no running task found: task 738ce9c87b91202f857e66e9a392377298a3f47dead219804fcd1807091fc590 not found: not found
+ rm /tmp/tmp.2wmbOyc5yt /tmp/tmp.KCbdK1R1Xh
+ return 0
2025-10-09 19:05:05      14497 filelist.pbm
+ '[' -z '' ']'
+ run_restore backup-aws-s3-sharded _restore_sharded
+ local backup_name=backup-aws-s3-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-10-09T19:05:13+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dbtohow8Yk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JPwMGqAR5V
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dbtohow8Yk
++ cat /tmp/tmp.JPwMGqAR5V
++ rm /tmp/tmp.dbtohow8Yk /tmp/tmp.JPwMGqAR5V
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mhaIPRGLhy
++ mktemp
+ local LAST_ERR=/tmp/tmp.Gw6Hwzersc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mhaIPRGLhy
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("52a4f228-3d50-4925-ac97-504b54f178a4") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.Gw6Hwzersc
+ rm /tmp/tmp.mhaIPRGLhy /tmp/tmp.Gw6Hwzersc
+ return 0
+ log 'check backup and restore -- backup-aws-s3-sharded'
+ set +o xtrace
[2025-10-09T19:05:16+0000] check backup and restore -- backup-aws-s3-sharded
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3-sharded/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.HBaNZlaDxH
++ mktemp
+ local LAST_ERR=/tmp/tmp.I7pyzaSOwZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HBaNZlaDxH
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3-sharded created
+ cat /tmp/tmp.I7pyzaSOwZ
+ rm /tmp/tmp.HBaNZlaDxH /tmp/tmp.I7pyzaSOwZ
+ return 0
+ run_recovery_check backup-aws-s3-sharded _restore_sharded
+ local backup_name=backup-aws-s3-sharded
+ local compare_suffix=_restore_sharded
+ local base=true
+ wait_restore backup-aws-s3-sharded some-name requested 0 3000
+ local backup_name=backup-aws-s3-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-aws-s3-sharded to reach state "requested" .....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-19753", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.giuGoKdzFf
++ mktemp
+ local LAST_ERR=/tmp/tmp.4Tuc7QdqZ4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.giuGoKdzFf
+ cat /tmp/tmp.4Tuc7QdqZ4
+ rm /tmp/tmp.giuGoKdzFf /tmp/tmp.4Tuc7QdqZ4
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-10-09T19:09:40+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ wait_restore backup-aws-s3-sharded some-name ready 0 3000
+ local backup_name=backup-aws-s3-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-aws-s3-sharded to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.l0OZBLYDsf
++ mktemp
+ local LAST_ERR=/tmp/tmp.ioKnplLWNv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.l0OZBLYDsf
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-incremental-sharded-19753"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-server-mongodb-operator:main-backup","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-incremental-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"minio":{"main":true,"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
    percona.com/resync-pbm: "true"
  creationTimestamp: "2025-10-09T18:57:39Z"
  generation: 2
  name: some-name
  namespace: demand-backup-incremental-sharded-19753
  resourceVersion: "1760037144871935024"
  uid: 90d40a75-a69f-432c-b6ab-5c963b6c67bc
spec:
  backup:
    enabled: true
    image: perconalab/percona-server-mongodb-operator:main-backup
    storages:
      aws-s3:
        s3:
          bucket: operator-testing
          credentialsSecret: aws-s3-secret
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      azure-blob:
        azure:
          container: operator-testing
          credentialsSecret: azure-secret
          prefix: psmdb-demand-backup-incremental-sharded
        type: azure
      gcp-cs:
        s3:
          bucket: operator-testing
          credentialsSecret: gcp-cs-secret
          endpointUrl: https://storage.googleapis.com
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      minio:
        main: true
        s3:
          bucket: operator-testing
          credentialsSecret: minio-secret
          endpointUrl: http://minio-service:9000/
          insecureSkipTLSVerify: false
          region: us-east-1
        type: s3
    tasks:
    - compressionType: gzip
      enabled: true
      name: weekly
      schedule: 0 0 * * 0
      storageName: aws-s3
  crVersion: 1.21.0
  image: perconalab/percona-server-mongodb-operator:main-mongod7.0
  imagePullPolicy: Always
  replsets:
  - affinity:
      antiAffinityTopologyKey: none
    configuration: |
      operationProfiling:
        mode: slowOp
        slowOpThresholdMs: 100
      security:
        enableEncryption: true
        redactClientLogData: false
      setParameter:
        ttlMonitorSleepSecs: 60
        wiredTigerConcurrentReadTransactions: 128
        wiredTigerConcurrentWriteTransactions: 128
      storage:
        engine: wiredTiger
        wiredTiger:
          collectionConfig:
            blockCompressor: snappy
          engineConfig:
            directoryForIndexes: false
            journalCompressor: snappy
          indexConfig:
            prefixCompression: true
    expose:
      enabled: false
      type: ClusterIP
    name: rs0
    resources:
      limits:
        cpu: 500m
        memory: 1G
      requests:
        cpu: 100m
        memory: 0.1G
    size: 3
    volumeSpec:
      persistentVolumeClaim:
        resources:
          requests:
            storage: 3Gi
  secrets:
    users: some-users
  sharding:
    configsvrReplSet:
      affinity:
        antiAffinityTopologyKey: none
      configuration: |
        operationProfiling:
          mode: slowOp
          slowOpThresholdMs: 100
        security:
          enableEncryption: true
          redactClientLogData: false
        setParameter:
          ttlMonitorSleepSecs: 60
          wiredTigerConcurrentReadTransactions: 128
          wiredTigerConcurrentWriteTransactions: 128
        storage:
          engine: wiredTiger
          wiredTiger:
            collectionConfig:
              blockCompressor: snappy
            engineConfig:
              directoryForIndexes: false
              journalCompressor: snappy
            indexConfig:
              prefixCompression: true
      expose:
        enabled: false
        type: ClusterIP
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
      volumeSpec:
        persistentVolumeClaim:
          resources:
            requests:
              storage: 3Gi
    enabled: true
    mongos:
      affinity:
        antiAffinityTopologyKey: none
      expose:
        type: LoadBalancer
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
  upgradeOptions:
    apply: Never
status:
  backupConfigHash: 657556472134be8a3e4b574eeba7d6db2138970454efb713e0add6290c72d12f
  backupImage: perconalab/percona-server-mongodb-operator:main-backup
  backupVersion: 2.11.0
  conditions:
  - lastTransitionTime: "2025-10-09T18:57:39Z"
    status: "True"
    type: sharding
  - lastTransitionTime: "2025-10-09T18:57:43Z"
    status: "True"
    type: initializing
  - lastTransitionTime: "2025-10-09T18:59:19Z"
    message: 'rs0: ready'
    reason: RSReady
    status: "True"
    type: ready
  host: 34.58.224.177
  mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
  mongoVersion: 7.0.24-13
  mongos:
    ready: 0
    size: 0
    status: initializing
  observedGeneration: 2
  ready: 6
  replsets:
    cfg:
      initialized: true
      ready: 3
      size: 3
      status: ready
    rs0:
      added_as_shard: true
      initialized: true
      ready: 3
      size: 3
      status: ready
  size: 6
  state: initializing
+ cat /tmp/tmp.ioKnplLWNv
+ rm /tmp/tmp.l0OZBLYDsf /tmp/tmp.ioKnplLWNv
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LR6RKL4DD2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FKnqWvR9Li
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LR6RKL4DD2
++ cat /tmp/tmp.FKnqWvR9Li
++ rm /tmp/tmp.LR6RKL4DD2 /tmp/tmp.FKnqWvR9Li
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 60
+ local cluster_name=some-name
+ local wait_time=60
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uHGUVBHKaL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jTkf3hxQof
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uHGUVBHKaL
++ cat /tmp/tmp.jTkf3hxQof
++ rm /tmp/tmp.uHGUVBHKaL /tmp/tmp.jTkf3hxQof
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fdAGwfrrKO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X9rt7QycoG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fdAGwfrrKO
++ cat /tmp/tmp.X9rt7QycoG
++ rm /tmp/tmp.fdAGwfrrKO /tmp/tmp.X9rt7QycoG
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zJsIT4kAJs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MApF0btefY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zJsIT4kAJs
++ cat /tmp/tmp.MApF0btefY
++ rm /tmp/tmp.zJsIT4kAJs /tmp/tmp.MApF0btefY
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pSb4vZ6YCu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sKSF6Ocre7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pSb4vZ6YCu
++ cat /tmp/tmp.sKSF6Ocre7
++ rm /tmp/tmp.pSb4vZ6YCu /tmp/tmp.sKSF6Ocre7
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AxccKrKYwf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9xfY2RCOjk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AxccKrKYwf
++ cat /tmp/tmp.9xfY2RCOjk
++ rm /tmp/tmp.AxccKrKYwf /tmp/tmp.9xfY2RCOjk
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CrwVbdyr17
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5Wh1XMj5xY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CrwVbdyr17
++ cat /tmp/tmp.5Wh1XMj5xY
++ rm /tmp/tmp.CrwVbdyr17 /tmp/tmp.5Wh1XMj5xY
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.30bGjkzyUn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DxSQ4RHaOn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.30bGjkzyUn
++ cat /tmp/tmp.DxSQ4RHaOn
++ rm /tmp/tmp.30bGjkzyUn /tmp/tmp.DxSQ4RHaOn
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1u7yFHAlQU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kOo7D24hP0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1u7yFHAlQU
++ cat /tmp/tmp.kOo7D24hP0
++ rm /tmp/tmp.1u7yFHAlQU /tmp/tmp.kOo7D24hP0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nNlRrWueZj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nTGVaGML8I
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nNlRrWueZj
++ cat /tmp/tmp.nTGVaGML8I
++ rm /tmp/tmp.nNlRrWueZj /tmp/tmp.nTGVaGML8I
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WDG2iB7vpd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.biBoWqNuMt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WDG2iB7vpd
++ cat /tmp/tmp.biBoWqNuMt
++ rm /tmp/tmp.WDG2iB7vpd /tmp/tmp.biBoWqNuMt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UT89bHOieR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y7BWU466n1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UT89bHOieR
++ cat /tmp/tmp.y7BWU466n1
++ rm /tmp/tmp.UT89bHOieR /tmp/tmp.y7BWU466n1
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish......OK
+ [[ true == true ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TrPIhb3frw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FwIYNdThSz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TrPIhb3frw
++ cat /tmp/tmp.FwIYNdThSz
++ rm /tmp/tmp.TrPIhb3frw /tmp/tmp.FwIYNdThSz
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4nkzl1zA4M
++ mktemp
+ local LAST_ERR=/tmp/tmp.pOrEQU1q1R
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4nkzl1zA4M
+ cat /tmp/tmp.pOrEQU1q1R
+ rm /tmp/tmp.4nkzl1zA4M /tmp/tmp.pOrEQU1q1R
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.3UJ4bnYUHf/find-sharded
+ check_exported_mongos_service_endpoint 34.58.224.177
+ local host=34.58.224.177
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YLqU2dDvh4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4DBG1P7Ukv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YLqU2dDvh4
++ cat /tmp/tmp.4DBG1P7Ukv
++ rm /tmp/tmp.YLqU2dDvh4 /tmp/tmp.4DBG1P7Ukv
++ return 0
+ '[' 34.58.224.177 '!=' 34.58.224.177 ']'
+ run_restore backup-gcp-cs-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-10-09T19:15:07+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8oYYzokCsG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aH707oUEq9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8oYYzokCsG
++ cat /tmp/tmp.aH707oUEq9
++ rm /tmp/tmp.8oYYzokCsG /tmp/tmp.aH707oUEq9
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pOcA9qAVSk
++ mktemp
+ local LAST_ERR=/tmp/tmp.QUbB7ObH0I
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pOcA9qAVSk
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b467bc19-b779-4242-98b0-2c88e8cf2ad0") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.QUbB7ObH0I
+ rm /tmp/tmp.pOcA9qAVSk /tmp/tmp.QUbB7ObH0I
+ return 0
+ log 'check backup and restore -- backup-gcp-cs-sharded'
+ set +o xtrace
[2025-10-09T19:15:09+0000] check backup and restore -- backup-gcp-cs-sharded
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sharded/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.NoMbXeHg0l
++ mktemp
+ local LAST_ERR=/tmp/tmp.XXfUkBwzUU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NoMbXeHg0l
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sharded created
+ cat /tmp/tmp.XXfUkBwzUU
+ rm /tmp/tmp.NoMbXeHg0l /tmp/tmp.XXfUkBwzUU
+ return 0
+ run_recovery_check backup-gcp-cs-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sharded
+ local compare_suffix=_restore_sharded
+ local base=true
+ wait_restore backup-gcp-cs-sharded some-name requested 0 3000
+ local backup_name=backup-gcp-cs-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-sharded to reach state "requested" .....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-19753", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.BRspfm4hwB
++ mktemp
+ local LAST_ERR=/tmp/tmp.RZH67SCWUJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BRspfm4hwB
+ cat /tmp/tmp.RZH67SCWUJ
+ rm /tmp/tmp.BRspfm4hwB /tmp/tmp.RZH67SCWUJ
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-10-09T19:19:40+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ wait_restore backup-gcp-cs-sharded some-name ready 0 3000
+ local backup_name=backup-gcp-cs-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-sharded to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.LsF6sixzgO
++ mktemp
+ local LAST_ERR=/tmp/tmp.iDYJYuqUIH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LsF6sixzgO
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-incremental-sharded-19753"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-server-mongodb-operator:main-backup","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-incremental-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"minio":{"main":true,"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
    percona.com/resync-pbm: "true"
  creationTimestamp: "2025-10-09T18:57:39Z"
  generation: 2
  name: some-name
  namespace: demand-backup-incremental-sharded-19753
  resourceVersion: "1760037733882335024"
  uid: 90d40a75-a69f-432c-b6ab-5c963b6c67bc
spec:
  backup:
    enabled: true
    image: perconalab/percona-server-mongodb-operator:main-backup
    storages:
      aws-s3:
        s3:
          bucket: operator-testing
          credentialsSecret: aws-s3-secret
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      azure-blob:
        azure:
          container: operator-testing
          credentialsSecret: azure-secret
          prefix: psmdb-demand-backup-incremental-sharded
        type: azure
      gcp-cs:
        s3:
          bucket: operator-testing
          credentialsSecret: gcp-cs-secret
          endpointUrl: https://storage.googleapis.com
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      minio:
        main: true
        s3:
          bucket: operator-testing
          credentialsSecret: minio-secret
          endpointUrl: http://minio-service:9000/
          insecureSkipTLSVerify: false
          region: us-east-1
        type: s3
    tasks:
    - compressionType: gzip
      enabled: true
      name: weekly
      schedule: 0 0 * * 0
      storageName: aws-s3
  crVersion: 1.21.0
  image: perconalab/percona-server-mongodb-operator:main-mongod7.0
  imagePullPolicy: Always
  replsets:
  - affinity:
      antiAffinityTopologyKey: none
    configuration: |
      operationProfiling:
        mode: slowOp
        slowOpThresholdMs: 100
      security:
        enableEncryption: true
        redactClientLogData: false
      setParameter:
        ttlMonitorSleepSecs: 60
        wiredTigerConcurrentReadTransactions: 128
        wiredTigerConcurrentWriteTransactions: 128
      storage:
        engine: wiredTiger
        wiredTiger:
          collectionConfig:
            blockCompressor: snappy
          engineConfig:
            directoryForIndexes: false
            journalCompressor: snappy
          indexConfig:
            prefixCompression: true
    expose:
      enabled: false
      type: ClusterIP
    name: rs0
    resources:
      limits:
        cpu: 500m
        memory: 1G
      requests:
        cpu: 100m
        memory: 0.1G
    size: 3
    volumeSpec:
      persistentVolumeClaim:
        resources:
          requests:
            storage: 3Gi
  secrets:
    users: some-users
  sharding:
    configsvrReplSet:
      affinity:
        antiAffinityTopologyKey: none
      configuration: |
        operationProfiling:
          mode: slowOp
          slowOpThresholdMs: 100
        security:
          enableEncryption: true
          redactClientLogData: false
        setParameter:
          ttlMonitorSleepSecs: 60
          wiredTigerConcurrentReadTransactions: 128
          wiredTigerConcurrentWriteTransactions: 128
        storage:
          engine: wiredTiger
          wiredTiger:
            collectionConfig:
              blockCompressor: snappy
            engineConfig:
              directoryForIndexes: false
              journalCompressor: snappy
            indexConfig:
              prefixCompression: true
      expose:
        enabled: false
        type: ClusterIP
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
      volumeSpec:
        persistentVolumeClaim:
          resources:
            requests:
              storage: 3Gi
    enabled: true
    mongos:
      affinity:
        antiAffinityTopologyKey: none
      expose:
        type: LoadBalancer
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
  upgradeOptions:
    apply: Never
status:
  backupConfigHash: 657556472134be8a3e4b574eeba7d6db2138970454efb713e0add6290c72d12f
  backupImage: perconalab/percona-server-mongodb-operator:main-backup
  backupVersion: 2.11.0
  conditions:
  - lastTransitionTime: "2025-10-09T18:57:39Z"
    status: "True"
    type: sharding
  - lastTransitionTime: "2025-10-09T18:57:43Z"
    status: "True"
    type: initializing
  - lastTransitionTime: "2025-10-09T18:59:19Z"
    message: 'rs0: ready'
    reason: RSReady
    status: "True"
    type: ready
  - lastTransitionTime: "2025-10-09T19:13:15Z"
    message: 'reconcile replset cfg: reconcile StatefulSet for cfg: failed to run
      smartUpdate: failed to check active jobs: getting PBM object: create PBM connection
      to some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-19753.svc.cluster.local:27017:
      create mongo connection: ping: server selection error: server selection timeout,
      current topology: { Type: Unknown, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-19753.svc.cluster.local:27017,
      Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-19753.svc.cluster.local
      on 34.118.224.10:53: no such host }, ] }'
    reason: ErrorReconcile
    status: "True"
    type: error
  host: 34.58.224.177
  mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
  mongoVersion: 7.0.24-13
  mongos:
    ready: 0
    size: 0
    status: initializing
  observedGeneration: 2
  ready: 6
  replsets:
    cfg:
      initialized: true
      ready: 3
      size: 3
      status: ready
    rs0:
      added_as_shard: true
      initialized: true
      ready: 3
      size: 3
      status: ready
  size: 6
  state: initializing
+ cat /tmp/tmp.iDYJYuqUIH
+ rm /tmp/tmp.LsF6sixzgO /tmp/tmp.iDYJYuqUIH
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fb4XtV4XC8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rLGvglpjtT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fb4XtV4XC8
++ cat /tmp/tmp.rLGvglpjtT
++ rm /tmp/tmp.fb4XtV4XC8 /tmp/tmp.rLGvglpjtT
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 60
+ local cluster_name=some-name
+ local wait_time=60
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eGExaJ7d2R
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vlLwJqemF6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eGExaJ7d2R
++ cat /tmp/tmp.vlLwJqemF6
++ rm /tmp/tmp.eGExaJ7d2R /tmp/tmp.vlLwJqemF6
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FXTvAFxZ73
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AWl35mqqoC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FXTvAFxZ73
++ cat /tmp/tmp.AWl35mqqoC
++ rm /tmp/tmp.FXTvAFxZ73 /tmp/tmp.AWl35mqqoC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZNRrCGcOWY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kDCkhrY7I7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZNRrCGcOWY
++ cat /tmp/tmp.kDCkhrY7I7
++ rm /tmp/tmp.ZNRrCGcOWY /tmp/tmp.kDCkhrY7I7
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bjGJ0D5jrD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.V4AcA9bJ43
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bjGJ0D5jrD
++ cat /tmp/tmp.V4AcA9bJ43
++ rm /tmp/tmp.bjGJ0D5jrD /tmp/tmp.V4AcA9bJ43
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.26fJtrti7L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wDPPUIAp3x
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.26fJtrti7L
++ cat /tmp/tmp.wDPPUIAp3x
++ rm /tmp/tmp.26fJtrti7L /tmp/tmp.wDPPUIAp3x
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZmtizW1NMo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1f1Rg9Tw1E
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZmtizW1NMo
++ cat /tmp/tmp.1f1Rg9Tw1E
++ rm /tmp/tmp.ZmtizW1NMo /tmp/tmp.1f1Rg9Tw1E
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IJ0l9tVQCk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VwbC3bavKC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IJ0l9tVQCk
++ cat /tmp/tmp.VwbC3bavKC
++ rm /tmp/tmp.IJ0l9tVQCk /tmp/tmp.VwbC3bavKC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l6zougHdSR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QnHRZcBEAn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.l6zougHdSR
++ cat /tmp/tmp.QnHRZcBEAn
++ rm /tmp/tmp.l6zougHdSR /tmp/tmp.QnHRZcBEAn
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aakbSHxkCv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EYJPGCmR5K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aakbSHxkCv
++ cat /tmp/tmp.EYJPGCmR5K
++ rm /tmp/tmp.aakbSHxkCv /tmp/tmp.EYJPGCmR5K
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.n7Fj31x1X9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jDJ1nPhYLq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.n7Fj31x1X9
++ cat /tmp/tmp.jDJ1nPhYLq
++ rm /tmp/tmp.n7Fj31x1X9 /tmp/tmp.jDJ1nPhYLq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ivlNSntwo1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Rbi8Jbr3vN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ivlNSntwo1
++ cat /tmp/tmp.Rbi8Jbr3vN
++ rm /tmp/tmp.ivlNSntwo1 /tmp/tmp.Rbi8Jbr3vN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oKlaW7mJNT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0eEyrKy1vs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oKlaW7mJNT
++ cat /tmp/tmp.0eEyrKy1vs
++ rm /tmp/tmp.oKlaW7mJNT /tmp/tmp.0eEyrKy1vs
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pcMPHmslyS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ua3FZkYNlV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pcMPHmslyS
++ cat /tmp/tmp.ua3FZkYNlV
++ rm /tmp/tmp.pcMPHmslyS /tmp/tmp.ua3FZkYNlV
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hXili1Kmml
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xLxuYdo45x
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hXili1Kmml
++ cat /tmp/tmp.xLxuYdo45x
++ rm /tmp/tmp.hXili1Kmml /tmp/tmp.xLxuYdo45x
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish......OK
+ [[ true == true ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O2MeT2J2uw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.z9xFkMK0lu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.O2MeT2J2uw
++ cat /tmp/tmp.z9xFkMK0lu
++ rm /tmp/tmp.O2MeT2J2uw /tmp/tmp.z9xFkMK0lu
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4FPMhyNGia
++ mktemp
+ local LAST_ERR=/tmp/tmp.USAKHD9Nai
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4FPMhyNGia
+ cat /tmp/tmp.USAKHD9Nai
+ rm /tmp/tmp.4FPMhyNGia /tmp/tmp.USAKHD9Nai
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.3UJ4bnYUHf/find-sharded
+ check_exported_mongos_service_endpoint 34.58.224.177
+ local host=34.58.224.177
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1ZjyvvXfu1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Xz1YBS6Pn7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1ZjyvvXfu1
++ cat /tmp/tmp.Xz1YBS6Pn7
++ rm /tmp/tmp.1ZjyvvXfu1 /tmp/tmp.Xz1YBS6Pn7
++ return 0
+ '[' 34.58.224.177 '!=' 34.58.224.177 ']'
+ run_restore backup-azure-blob-sharded _restore_sharded
+ local backup_name=backup-azure-blob-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-10-09T19:25:28+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iTOgxGxxZO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.daunWwNLFa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iTOgxGxxZO
++ cat /tmp/tmp.daunWwNLFa
++ rm /tmp/tmp.iTOgxGxxZO /tmp/tmp.daunWwNLFa
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5Fxqk28QBB
++ mktemp
+ local LAST_ERR=/tmp/tmp.CugJdIgQ2f
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5Fxqk28QBB
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("60447006-8e80-48b1-8b09-d48a383d5841") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.CugJdIgQ2f
+ rm /tmp/tmp.5Fxqk28QBB /tmp/tmp.CugJdIgQ2f
+ return 0
+ log 'check backup and restore -- backup-azure-blob-sharded'
+ set +o xtrace
[2025-10-09T19:25:32+0000] check backup and restore -- backup-azure-blob-sharded
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob-sharded/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.QRxyYLxTrR
++ mktemp
+ local LAST_ERR=/tmp/tmp.ifp5QK9r2c
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QRxyYLxTrR
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob-sharded created
+ cat /tmp/tmp.ifp5QK9r2c
+ rm /tmp/tmp.QRxyYLxTrR /tmp/tmp.ifp5QK9r2c
+ return 0
+ run_recovery_check backup-azure-blob-sharded _restore_sharded
+ local backup_name=backup-azure-blob-sharded
+ local compare_suffix=_restore_sharded
+ local base=true
+ wait_restore backup-azure-blob-sharded some-name requested 0 3000
+ local backup_name=backup-azure-blob-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-azure-blob-sharded to reach state "requested" .....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-19753", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.hcRXLiZvJh
++ mktemp
+ local LAST_ERR=/tmp/tmp.AgsgVD4m3M
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hcRXLiZvJh
+ cat /tmp/tmp.AgsgVD4m3M
+ rm /tmp/tmp.hcRXLiZvJh /tmp/tmp.AgsgVD4m3M
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-10-09T19:29:54+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ wait_restore backup-azure-blob-sharded some-name ready 0 3000
+ local backup_name=backup-azure-blob-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-azure-blob-sharded to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.J9wUuZNraN
++ mktemp
+ local LAST_ERR=/tmp/tmp.XqwniKS8NG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J9wUuZNraN
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-incremental-sharded-19753"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-server-mongodb-operator:main-backup","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-incremental-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"minio":{"main":true,"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
    percona.com/resync-pbm: "true"
  creationTimestamp: "2025-10-09T18:57:39Z"
  generation: 2
  name: some-name
  namespace: demand-backup-incremental-sharded-19753
  resourceVersion: "1760038352611503024"
  uid: 90d40a75-a69f-432c-b6ab-5c963b6c67bc
spec:
  backup:
    enabled: true
    image: perconalab/percona-server-mongodb-operator:main-backup
    storages:
      aws-s3:
        s3:
          bucket: operator-testing
          credentialsSecret: aws-s3-secret
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      azure-blob:
        azure:
          container: operator-testing
          credentialsSecret: azure-secret
          prefix: psmdb-demand-backup-incremental-sharded
        type: azure
      gcp-cs:
        s3:
          bucket: operator-testing
          credentialsSecret: gcp-cs-secret
          endpointUrl: https://storage.googleapis.com
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      minio:
        main: true
        s3:
          bucket: operator-testing
          credentialsSecret: minio-secret
          endpointUrl: http://minio-service:9000/
          insecureSkipTLSVerify: false
          region: us-east-1
        type: s3
    tasks:
    - compressionType: gzip
      enabled: true
      name: weekly
      schedule: 0 0 * * 0
      storageName: aws-s3
  crVersion: 1.21.0
  image: perconalab/percona-server-mongodb-operator:main-mongod7.0
  imagePullPolicy: Always
  replsets:
  - affinity:
      antiAffinityTopologyKey: none
    configuration: |
      operationProfiling:
        mode: slowOp
        slowOpThresholdMs: 100
      security:
        enableEncryption: true
        redactClientLogData: false
      setParameter:
        ttlMonitorSleepSecs: 60
        wiredTigerConcurrentReadTransactions: 128
        wiredTigerConcurrentWriteTransactions: 128
      storage:
        engine: wiredTiger
        wiredTiger:
          collectionConfig:
            blockCompressor: snappy
          engineConfig:
            directoryForIndexes: false
            journalCompressor: snappy
          indexConfig:
            prefixCompression: true
    expose:
      enabled: false
      type: ClusterIP
    name: rs0
    resources:
      limits:
        cpu: 500m
        memory: 1G
      requests:
        cpu: 100m
        memory: 0.1G
    size: 3
    volumeSpec:
      persistentVolumeClaim:
        resources:
          requests:
            storage: 3Gi
  secrets:
    users: some-users
  sharding:
    configsvrReplSet:
      affinity:
        antiAffinityTopologyKey: none
      configuration: |
        operationProfiling:
          mode: slowOp
          slowOpThresholdMs: 100
        security:
          enableEncryption: true
          redactClientLogData: false
        setParameter:
          ttlMonitorSleepSecs: 60
          wiredTigerConcurrentReadTransactions: 128
          wiredTigerConcurrentWriteTransactions: 128
        storage:
          engine: wiredTiger
          wiredTiger:
            collectionConfig:
              blockCompressor: snappy
            engineConfig:
              directoryForIndexes: false
              journalCompressor: snappy
            indexConfig:
              prefixCompression: true
      expose:
        enabled: false
        type: ClusterIP
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
      volumeSpec:
        persistentVolumeClaim:
          resources:
            requests:
              storage: 3Gi
    enabled: true
    mongos:
      affinity:
        antiAffinityTopologyKey: none
      expose:
        type: LoadBalancer
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
  upgradeOptions:
    apply: Never
status:
  backupConfigHash: 657556472134be8a3e4b574eeba7d6db2138970454efb713e0add6290c72d12f
  backupImage: perconalab/percona-server-mongodb-operator:main-backup
  backupVersion: 2.11.0
  conditions:
  - lastTransitionTime: "2025-10-09T18:57:39Z"
    status: "True"
    type: sharding
  - lastTransitionTime: "2025-10-09T18:57:43Z"
    status: "True"
    type: initializing
  - lastTransitionTime: "2025-10-09T18:59:19Z"
    message: 'rs0: ready'
    reason: RSReady
    status: "True"
    type: ready
  - lastTransitionTime: "2025-10-09T19:13:15Z"
    message: |-
      dial: ping mongo: server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-19753.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-19753.svc.cluster.local on 34.118.224.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.demand-backup-incremental-sharded-19753.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 10.183.34.41:27017: connect: connection refused }, { Addr: some-name-cfg-2.some-name-cfg.demand-backup-incremental-sharded-19753.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 10.183.32.47:27017: connect: connection refused }, ] }
      handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-19753.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-19753.svc.cluster.local on 34.118.224.10:53: no such host }, ] }
    reason: ErrorReconcile
    status: "True"
    type: error
  host: 34.58.224.177
  mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
  mongoVersion: 7.0.24-13
  mongos:
    ready: 0
    size: 0
    status: initializing
  observedGeneration: 2
  ready: 6
  replsets:
    cfg:
      initialized: true
      ready: 3
      size: 3
      status: ready
    rs0:
      added_as_shard: true
      initialized: true
      ready: 3
      size: 3
      status: ready
  size: 6
  state: initializing
+ cat /tmp/tmp.XqwniKS8NG
+ rm /tmp/tmp.J9wUuZNraN /tmp/tmp.XqwniKS8NG
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kZjqTnwaby
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RMZrwN2XT0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kZjqTnwaby
++ cat /tmp/tmp.RMZrwN2XT0
++ rm /tmp/tmp.kZjqTnwaby /tmp/tmp.RMZrwN2XT0
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 60
+ local cluster_name=some-name
+ local wait_time=60
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cYSqghN1cp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oe0IDGA9pZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cYSqghN1cp
++ cat /tmp/tmp.oe0IDGA9pZ
++ rm /tmp/tmp.cYSqghN1cp /tmp/tmp.oe0IDGA9pZ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6APYej4cA1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fLp0xdkERX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6APYej4cA1
++ cat /tmp/tmp.fLp0xdkERX
++ rm /tmp/tmp.6APYej4cA1 /tmp/tmp.fLp0xdkERX
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F5tQfXyRUh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BggJe5VvoU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.F5tQfXyRUh
++ cat /tmp/tmp.BggJe5VvoU
++ rm /tmp/tmp.F5tQfXyRUh /tmp/tmp.BggJe5VvoU
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JqMTXyBFfD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SCOKbh9o9u
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JqMTXyBFfD
++ cat /tmp/tmp.SCOKbh9o9u
++ rm /tmp/tmp.JqMTXyBFfD /tmp/tmp.SCOKbh9o9u
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MO5TXT2E7i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jR6nbeW0LF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MO5TXT2E7i
++ cat /tmp/tmp.jR6nbeW0LF
++ rm /tmp/tmp.MO5TXT2E7i /tmp/tmp.jR6nbeW0LF
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Kmj3a4jZrA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sA9cJoUFNI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Kmj3a4jZrA
++ cat /tmp/tmp.sA9cJoUFNI
++ rm /tmp/tmp.Kmj3a4jZrA /tmp/tmp.sA9cJoUFNI
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WvikkFoUyP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vPD3lFZmUn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WvikkFoUyP
++ cat /tmp/tmp.vPD3lFZmUn
++ rm /tmp/tmp.WvikkFoUyP /tmp/tmp.vPD3lFZmUn
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4o1DVT8SU0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OJtoPPmWxR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4o1DVT8SU0
++ cat /tmp/tmp.OJtoPPmWxR
++ rm /tmp/tmp.4o1DVT8SU0 /tmp/tmp.OJtoPPmWxR
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.o8xL2JV7Xm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UPSmQ0ExJE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.o8xL2JV7Xm
++ cat /tmp/tmp.UPSmQ0ExJE
++ rm /tmp/tmp.o8xL2JV7Xm /tmp/tmp.UPSmQ0ExJE
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NysTDkzRd8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.D97E57rjHD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NysTDkzRd8
++ cat /tmp/tmp.D97E57rjHD
++ rm /tmp/tmp.NysTDkzRd8 /tmp/tmp.D97E57rjHD
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WR3bkP3hjP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lZYQPEr0dd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WR3bkP3hjP
++ cat /tmp/tmp.lZYQPEr0dd
++ rm /tmp/tmp.WR3bkP3hjP /tmp/tmp.lZYQPEr0dd
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish.......OK
+ [[ true == true ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ubPGhv90ek
+++ mktemp
++ local LAST_ERR=/tmp/tmp.j1Ke0nAuBC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ubPGhv90ek
++ cat /tmp/tmp.j1Ke0nAuBC
++ rm /tmp/tmp.ubPGhv90ek /tmp/tmp.j1Ke0nAuBC
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VQKmdsWzET
++ mktemp
+ local LAST_ERR=/tmp/tmp.rCgrI7rDqt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VQKmdsWzET
+ cat /tmp/tmp.rCgrI7rDqt
+ rm /tmp/tmp.VQKmdsWzET /tmp/tmp.rCgrI7rDqt
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.3UJ4bnYUHf/find-sharded
+ check_exported_mongos_service_endpoint 34.58.224.177
+ local host=34.58.224.177
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LfNWz8qbmC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FAkob56q7g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LfNWz8qbmC
++ cat /tmp/tmp.FAkob56q7g
++ rm /tmp/tmp.LfNWz8qbmC /tmp/tmp.FAkob56q7g
++ return 0
+ '[' 34.58.224.177 '!=' 34.58.224.177 ']'
+ run_restore backup-minio-not-base-sharded _restore_sharded
+ local backup_name=backup-minio-not-base-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-10-09T19:35:20+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ynnYuj8AYh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VGstA8FC1q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ynnYuj8AYh
++ cat /tmp/tmp.VGstA8FC1q
++ rm /tmp/tmp.ynnYuj8AYh /tmp/tmp.VGstA8FC1q
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.TJtReEtBLY
++ mktemp
+ local LAST_ERR=/tmp/tmp.lDkOELDb7m
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TJtReEtBLY
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("65fb8959-c3e5-4ee1-a92b-50004765768d") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.lDkOELDb7m
+ rm /tmp/tmp.TJtReEtBLY /tmp/tmp.lDkOELDb7m
+ return 0
+ log 'check backup and restore -- backup-minio-not-base-sharded'
+ set +o xtrace
[2025-10-09T19:35:22+0000] check backup and restore -- backup-minio-not-base-sharded
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-not-base-sharded/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-not-base-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.tfm0i9AovD
++ mktemp
+ local LAST_ERR=/tmp/tmp.a0FEy1GPnh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tfm0i9AovD
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-not-base-sharded created
+ cat /tmp/tmp.a0FEy1GPnh
+ rm /tmp/tmp.tfm0i9AovD /tmp/tmp.a0FEy1GPnh
+ return 0
+ run_recovery_check backup-minio-not-base-sharded _restore_sharded false
+ local backup_name=backup-minio-not-base-sharded
+ local compare_suffix=_restore_sharded
+ local base=false
+ wait_restore backup-minio-not-base-sharded some-name requested 0 3000
+ local backup_name=backup-minio-not-base-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-not-base-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-minio-not-base-sharded to reach state "requested" .....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-19753", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.z4hLi4weTD
++ mktemp
+ local LAST_ERR=/tmp/tmp.hsCxpTLVOe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.z4hLi4weTD
+ cat /tmp/tmp.hsCxpTLVOe
+ rm /tmp/tmp.z4hLi4weTD /tmp/tmp.hsCxpTLVOe
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-10-09T19:39:50+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ wait_restore backup-minio-not-base-sharded some-name ready 0 3000
+ local backup_name=backup-minio-not-base-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-not-base-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-minio-not-base-sharded to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.OARRXES8Mf
++ mktemp
+ local LAST_ERR=/tmp/tmp.xj1zEGsWVY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OARRXES8Mf
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-incremental-sharded-19753"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-server-mongodb-operator:main-backup","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-incremental-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"minio":{"main":true,"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
    percona.com/resync-pbm: "true"
  creationTimestamp: "2025-10-09T18:57:39Z"
  generation: 2
  name: some-name
  namespace: demand-backup-incremental-sharded-19753
  resourceVersion: "1760038930265311024"
  uid: 90d40a75-a69f-432c-b6ab-5c963b6c67bc
spec:
  backup:
    enabled: true
    image: perconalab/percona-server-mongodb-operator:main-backup
    storages:
      aws-s3:
        s3:
          bucket: operator-testing
          credentialsSecret: aws-s3-secret
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      azure-blob:
        azure:
          container: operator-testing
          credentialsSecret: azure-secret
          prefix: psmdb-demand-backup-incremental-sharded
        type: azure
      gcp-cs:
        s3:
          bucket: operator-testing
          credentialsSecret: gcp-cs-secret
          endpointUrl: https://storage.googleapis.com
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      minio:
        main: true
        s3:
          bucket: operator-testing
          credentialsSecret: minio-secret
          endpointUrl: http://minio-service:9000/
          insecureSkipTLSVerify: false
          region: us-east-1
        type: s3
    tasks:
    - compressionType: gzip
      enabled: true
      name: weekly
      schedule: 0 0 * * 0
      storageName: aws-s3
  crVersion: 1.21.0
  image: perconalab/percona-server-mongodb-operator:main-mongod7.0
  imagePullPolicy: Always
  replsets:
  - affinity:
      antiAffinityTopologyKey: none
    configuration: |
      operationProfiling:
        mode: slowOp
        slowOpThresholdMs: 100
      security:
        enableEncryption: true
        redactClientLogData: false
      setParameter:
        ttlMonitorSleepSecs: 60
        wiredTigerConcurrentReadTransactions: 128
        wiredTigerConcurrentWriteTransactions: 128
      storage:
        engine: wiredTiger
        wiredTiger:
          collectionConfig:
            blockCompressor: snappy
          engineConfig:
            directoryForIndexes: false
            journalCompressor: snappy
          indexConfig:
            prefixCompression: true
    expose:
      enabled: false
      type: ClusterIP
    name: rs0
    resources:
      limits:
        cpu: 500m
        memory: 1G
      requests:
        cpu: 100m
        memory: 0.1G
    size: 3
    volumeSpec:
      persistentVolumeClaim:
        resources:
          requests:
            storage: 3Gi
  secrets:
    users: some-users
  sharding:
    configsvrReplSet:
      affinity:
        antiAffinityTopologyKey: none
      configuration: |
        operationProfiling:
          mode: slowOp
          slowOpThresholdMs: 100
        security:
          enableEncryption: true
          redactClientLogData: false
        setParameter:
          ttlMonitorSleepSecs: 60
          wiredTigerConcurrentReadTransactions: 128
          wiredTigerConcurrentWriteTransactions: 128
        storage:
          engine: wiredTiger
          wiredTiger:
            collectionConfig:
              blockCompressor: snappy
            engineConfig:
              directoryForIndexes: false
              journalCompressor: snappy
            indexConfig:
              prefixCompression: true
      expose:
        enabled: false
        type: ClusterIP
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
      volumeSpec:
        persistentVolumeClaim:
          resources:
            requests:
              storage: 3Gi
    enabled: true
    mongos:
      affinity:
        antiAffinityTopologyKey: none
      expose:
        type: LoadBalancer
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
  upgradeOptions:
    apply: Never
status:
  backupConfigHash: 657556472134be8a3e4b574eeba7d6db2138970454efb713e0add6290c72d12f
  backupImage: perconalab/percona-server-mongodb-operator:main-backup
  backupVersion: 2.11.0
  conditions:
  - lastTransitionTime: "2025-10-09T18:57:39Z"
    status: "True"
    type: sharding
  - lastTransitionTime: "2025-10-09T18:57:43Z"
    status: "True"
    type: initializing
  - lastTransitionTime: "2025-10-09T18:59:19Z"
    message: 'rs0: ready'
    reason: RSReady
    status: "True"
    type: ready
  - lastTransitionTime: "2025-10-09T19:13:15Z"
    message: 'handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo:
      server selection error: server selection timeout, current topology: { Type:
      Single, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-19753.svc.cluster.local:27017,
      Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-19753.svc.cluster.local
      on 34.118.224.10:53: no such host }, ] }'
    reason: ErrorReconcile
    status: "True"
    type: error
  host: 34.58.224.177
  mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
  mongoVersion: 7.0.24-13
  mongos:
    ready: 0
    size: 0
    status: initializing
  observedGeneration: 2
  ready: 6
  replsets:
    cfg:
      initialized: true
      ready: 3
      size: 3
      status: ready
    rs0:
      added_as_shard: true
      initialized: true
      ready: 3
      size: 3
      status: ready
  size: 6
  state: initializing
+ cat /tmp/tmp.xj1zEGsWVY
+ rm /tmp/tmp.OARRXES8Mf /tmp/tmp.xj1zEGsWVY
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.87GG8KdCsd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FZQBfLVdxk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.87GG8KdCsd
++ cat /tmp/tmp.FZQBfLVdxk
++ rm /tmp/tmp.87GG8KdCsd /tmp/tmp.FZQBfLVdxk
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 60
+ local cluster_name=some-name
+ local wait_time=60
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.x3WjVW7vzC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pedU3xfnNo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.x3WjVW7vzC
++ cat /tmp/tmp.pedU3xfnNo
++ rm /tmp/tmp.x3WjVW7vzC /tmp/tmp.pedU3xfnNo
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9ko8ymYyRz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vejvdJD8QR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9ko8ymYyRz
++ cat /tmp/tmp.vejvdJD8QR
++ rm /tmp/tmp.9ko8ymYyRz /tmp/tmp.vejvdJD8QR
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.muuUJfAZ7O
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9lWQ8XPKQs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.muuUJfAZ7O
++ cat /tmp/tmp.9lWQ8XPKQs
++ rm /tmp/tmp.muuUJfAZ7O /tmp/tmp.9lWQ8XPKQs
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ugCm6qnP6a
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BikRckFHwQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ugCm6qnP6a
++ cat /tmp/tmp.BikRckFHwQ
++ rm /tmp/tmp.ugCm6qnP6a /tmp/tmp.BikRckFHwQ
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UDpgFoQ8NA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QvD86ESI2a
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UDpgFoQ8NA
++ cat /tmp/tmp.QvD86ESI2a
++ rm /tmp/tmp.UDpgFoQ8NA /tmp/tmp.QvD86ESI2a
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rMh0FXg8bP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aQKSgbQHo2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rMh0FXg8bP
++ cat /tmp/tmp.aQKSgbQHo2
++ rm /tmp/tmp.rMh0FXg8bP /tmp/tmp.aQKSgbQHo2
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OT5CP5qxvu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lcZQGWKVid
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OT5CP5qxvu
++ cat /tmp/tmp.lcZQGWKVid
++ rm /tmp/tmp.OT5CP5qxvu /tmp/tmp.lcZQGWKVid
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pEJk67Ra5i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RTh8KnWbgi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pEJk67Ra5i
++ cat /tmp/tmp.RTh8KnWbgi
++ rm /tmp/tmp.pEJk67Ra5i /tmp/tmp.RTh8KnWbgi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TkGN731TQD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yA3IxGUj1g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TkGN731TQD
++ cat /tmp/tmp.yA3IxGUj1g
++ rm /tmp/tmp.TkGN731TQD /tmp/tmp.yA3IxGUj1g
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DONQqDLk19
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kgRWcIKx2H
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DONQqDLk19
++ cat /tmp/tmp.kgRWcIKx2H
++ rm /tmp/tmp.DONQqDLk19 /tmp/tmp.kgRWcIKx2H
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8brGIxIGdh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ICUr5J8Xdd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8brGIxIGdh
++ cat /tmp/tmp.ICUr5J8Xdd
++ rm /tmp/tmp.8brGIxIGdh /tmp/tmp.ICUr5J8Xdd
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4LgxcXl8ds
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GP75s36F3j
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4LgxcXl8ds
++ cat /tmp/tmp.GP75s36F3j
++ rm /tmp/tmp.4LgxcXl8ds /tmp/tmp.GP75s36F3j
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.e5dWApGv5f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g6bT0lxJZR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.e5dWApGv5f
++ cat /tmp/tmp.g6bT0lxJZR
++ rm /tmp/tmp.e5dWApGv5f /tmp/tmp.g6bT0lxJZR
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NMSyvP0jmu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rU0jUlHR5I
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NMSyvP0jmu
++ cat /tmp/tmp.rU0jUlHR5I
++ rm /tmp/tmp.NMSyvP0jmu /tmp/tmp.rU0jUlHR5I
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish.....OK
+ [[ false == true ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 -not-base-sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local postfix=-not-base-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
egrep: warning: egrep is obsolescent; using grep -E
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OgPSw8uUIa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1QDTiXRZVi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OgPSw8uUIa
++ cat /tmp/tmp.1QDTiXRZVi
++ rm /tmp/tmp.OgPSw8uUIa /tmp/tmp.1QDTiXRZVi
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7pdpug1uus
++ mktemp
+ local LAST_ERR=/tmp/tmp.H248B4k4mY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7pdpug1uus
+ cat /tmp/tmp.H248B4k4mY
+ rm /tmp/tmp.7pdpug1uus /tmp/tmp.H248B4k4mY
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/find-not-base-sharded.json /tmp/tmp.3UJ4bnYUHf/find-not-base-sharded
+ run_restore backup-minio-sharded _restore_sharded
+ local backup_name=backup-minio-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-10-09T19:45:17+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AnQePbtL14
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aI97ojFfa4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AnQePbtL14
++ cat /tmp/tmp.aI97ojFfa4
++ rm /tmp/tmp.AnQePbtL14 /tmp/tmp.aI97ojFfa4
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4qJDsP2mhR
++ mktemp
+ local LAST_ERR=/tmp/tmp.XOGhjjVcrl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4qJDsP2mhR
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("bab1a5c4-dd49-4ee1-afa0-5a43808a38b1") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.XOGhjjVcrl
+ rm /tmp/tmp.4qJDsP2mhR /tmp/tmp.XOGhjjVcrl
+ return 0
+ log 'check backup and restore -- backup-minio-sharded'
+ set +o xtrace
[2025-10-09T19:45:20+0000] check backup and restore -- backup-minio-sharded
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-sharded/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.dhBEFC3TZS
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZBfOMlwxaD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dhBEFC3TZS
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-sharded created
+ cat /tmp/tmp.ZBfOMlwxaD
+ rm /tmp/tmp.dhBEFC3TZS /tmp/tmp.ZBfOMlwxaD
+ return 0
+ run_recovery_check backup-minio-sharded _restore_sharded
+ local backup_name=backup-minio-sharded
+ local compare_suffix=_restore_sharded
+ local base=true
+ wait_restore backup-minio-sharded some-name requested 0 3000
+ local backup_name=backup-minio-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-minio-sharded to reach state "requested" .....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-19753", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fh8d93JwH5
++ mktemp
+ local LAST_ERR=/tmp/tmp.qcgzl4PYXY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fh8d93JwH5
+ cat /tmp/tmp.qcgzl4PYXY
+ rm /tmp/tmp.fh8d93JwH5 /tmp/tmp.qcgzl4PYXY
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.3UJ4bnYUHf/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-10-09T19:50:13+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ wait_restore backup-minio-sharded some-name ready 0 3000
+ local backup_name=backup-minio-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-minio-sharded to reach state "ready" ...OK after 2 minutes
+ [[ 0 -eq 1 ]]
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.bYGYEseZ5I
++ mktemp
+ local LAST_ERR=/tmp/tmp.JmtOhPVg1R
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bYGYEseZ5I
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-incremental-sharded-19753"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-server-mongodb-operator:main-backup","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-incremental-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-incremental-sharded","region":"us-east-1"},"type":"s3"},"minio":{"main":true,"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
    percona.com/resync-pbm: "true"
  creationTimestamp: "2025-10-09T18:57:39Z"
  generation: 2
  name: some-name
  namespace: demand-backup-incremental-sharded-19753
  resourceVersion: "1760039559908143024"
  uid: 90d40a75-a69f-432c-b6ab-5c963b6c67bc
spec:
  backup:
    enabled: true
    image: perconalab/percona-server-mongodb-operator:main-backup
    storages:
      aws-s3:
        s3:
          bucket: operator-testing
          credentialsSecret: aws-s3-secret
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      azure-blob:
        azure:
          container: operator-testing
          credentialsSecret: azure-secret
          prefix: psmdb-demand-backup-incremental-sharded
        type: azure
      gcp-cs:
        s3:
          bucket: operator-testing
          credentialsSecret: gcp-cs-secret
          endpointUrl: https://storage.googleapis.com
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-incremental-sharded
          region: us-east-1
        type: s3
      minio:
        main: true
        s3:
          bucket: operator-testing
          credentialsSecret: minio-secret
          endpointUrl: http://minio-service:9000/
          insecureSkipTLSVerify: false
          region: us-east-1
        type: s3
    tasks:
    - compressionType: gzip
      enabled: true
      name: weekly
      schedule: 0 0 * * 0
      storageName: aws-s3
  crVersion: 1.21.0
  image: perconalab/percona-server-mongodb-operator:main-mongod7.0
  imagePullPolicy: Always
  replsets:
  - affinity:
      antiAffinityTopologyKey: none
    configuration: |
      operationProfiling:
        mode: slowOp
        slowOpThresholdMs: 100
      security:
        enableEncryption: true
        redactClientLogData: false
      setParameter:
        ttlMonitorSleepSecs: 60
        wiredTigerConcurrentReadTransactions: 128
        wiredTigerConcurrentWriteTransactions: 128
      storage:
        engine: wiredTiger
        wiredTiger:
          collectionConfig:
            blockCompressor: snappy
          engineConfig:
            directoryForIndexes: false
            journalCompressor: snappy
          indexConfig:
            prefixCompression: true
    expose:
      enabled: false
      type: ClusterIP
    name: rs0
    resources:
      limits:
        cpu: 500m
        memory: 1G
      requests:
        cpu: 100m
        memory: 0.1G
    size: 3
    volumeSpec:
      persistentVolumeClaim:
        resources:
          requests:
            storage: 3Gi
  secrets:
    users: some-users
  sharding:
    configsvrReplSet:
      affinity:
        antiAffinityTopologyKey: none
      configuration: |
        operationProfiling:
          mode: slowOp
          slowOpThresholdMs: 100
        security:
          enableEncryption: true
          redactClientLogData: false
        setParameter:
          ttlMonitorSleepSecs: 60
          wiredTigerConcurrentReadTransactions: 128
          wiredTigerConcurrentWriteTransactions: 128
        storage:
          engine: wiredTiger
          wiredTiger:
            collectionConfig:
              blockCompressor: snappy
            engineConfig:
              directoryForIndexes: false
              journalCompressor: snappy
            indexConfig:
              prefixCompression: true
      expose:
        enabled: false
        type: ClusterIP
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
      volumeSpec:
        persistentVolumeClaim:
          resources:
            requests:
              storage: 3Gi
    enabled: true
    mongos:
      affinity:
        antiAffinityTopologyKey: none
      expose:
        type: LoadBalancer
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
  upgradeOptions:
    apply: Never
status:
  backupConfigHash: 657556472134be8a3e4b574eeba7d6db2138970454efb713e0add6290c72d12f
  backupImage: perconalab/percona-server-mongodb-operator:main-backup
  backupVersion: 2.11.0
  conditions:
  - lastTransitionTime: "2025-10-09T18:57:39Z"
    status: "True"
    type: sharding
  - lastTransitionTime: "2025-10-09T18:57:43Z"
    status: "True"
    type: initializing
  - lastTransitionTime: "2025-10-09T18:59:19Z"
    message: 'rs0: ready'
    reason: RSReady
    status: "True"
    type: ready
  - lastTransitionTime: "2025-10-09T19:13:15Z"
    message: 'handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo:
      server selection error: server selection timeout, current topology: { Type:
      Single, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-19753.svc.cluster.local:27017,
      Type: Unknown, Last error: dial tcp 10.183.33.69:27017: connect: connection
      refused }, ] }'
    reason: ErrorReconcile
    status: "True"
    type: error
  host: 34.58.224.177
  mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
  mongoVersion: 7.0.24-13
  mongos:
    ready: 0
    size: 0
    status: initializing
  observedGeneration: 2
  ready: 6
  replsets:
    cfg:
      initialized: true
      ready: 3
      size: 3
      status: ready
    rs0:
      added_as_shard: true
      initialized: true
      ready: 3
      size: 3
      status: ready
  size: 6
  state: initializing
+ cat /tmp/tmp.JmtOhPVg1R
+ rm /tmp/tmp.bYGYEseZ5I /tmp/tmp.JmtOhPVg1R
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cVLvXfrCG9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VVk9CjxpE7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cVLvXfrCG9
++ cat /tmp/tmp.VVk9CjxpE7
++ rm /tmp/tmp.cVLvXfrCG9 /tmp/tmp.VVk9CjxpE7
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 60
+ local cluster_name=some-name
+ local wait_time=60
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PQ5rKs5iPo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fs72fIZ0Ky
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PQ5rKs5iPo
++ cat /tmp/tmp.fs72fIZ0Ky
++ rm /tmp/tmp.PQ5rKs5iPo /tmp/tmp.fs72fIZ0Ky
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WndO7ez4Ws
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vLtLFMinmK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WndO7ez4Ws
++ cat /tmp/tmp.vLtLFMinmK
++ rm /tmp/tmp.WndO7ez4Ws /tmp/tmp.vLtLFMinmK
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9Z8E2EtoRr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.S7zHpcS0Dp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9Z8E2EtoRr
++ cat /tmp/tmp.S7zHpcS0Dp
++ rm /tmp/tmp.9Z8E2EtoRr /tmp/tmp.S7zHpcS0Dp
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Nb4BpOZtEG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tC02vmlRy1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Nb4BpOZtEG
++ cat /tmp/tmp.tC02vmlRy1
++ rm /tmp/tmp.Nb4BpOZtEG /tmp/tmp.tC02vmlRy1
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hY6EWkUMw4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kpZTsdqFOR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hY6EWkUMw4
++ cat /tmp/tmp.kpZTsdqFOR
++ rm /tmp/tmp.hY6EWkUMw4 /tmp/tmp.kpZTsdqFOR
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZCZVtpIgyF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MnAjDUW1e9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZCZVtpIgyF
++ cat /tmp/tmp.MnAjDUW1e9
++ rm /tmp/tmp.ZCZVtpIgyF /tmp/tmp.MnAjDUW1e9
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Pq4Pqu7ecu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nsZ4LDhTii
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Pq4Pqu7ecu
++ cat /tmp/tmp.nsZ4LDhTii
++ rm /tmp/tmp.Pq4Pqu7ecu /tmp/tmp.nsZ4LDhTii
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NqG1O07qfZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GrZ3wtYdCg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NqG1O07qfZ
++ cat /tmp/tmp.GrZ3wtYdCg
++ rm /tmp/tmp.NqG1O07qfZ /tmp/tmp.GrZ3wtYdCg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SH71bB7Gg2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4tFRNKX7M4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SH71bB7Gg2
++ cat /tmp/tmp.4tFRNKX7M4
++ rm /tmp/tmp.SH71bB7Gg2 /tmp/tmp.4tFRNKX7M4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gK4hxmZoaW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lsTYoRySO8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gK4hxmZoaW
++ cat /tmp/tmp.lsTYoRySO8
++ rm /tmp/tmp.gK4hxmZoaW /tmp/tmp.lsTYoRySO8
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.R1rYnIak3g
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LU5k1tqT1L
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.R1rYnIak3g
++ cat /tmp/tmp.LU5k1tqT1L
++ rm /tmp/tmp.R1rYnIak3g /tmp/tmp.LU5k1tqT1L
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7kwO51gXUb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MZ3DlHy0wy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7kwO51gXUb
++ cat /tmp/tmp.MZ3DlHy0wy
++ rm /tmp/tmp.7kwO51gXUb /tmp/tmp.MZ3DlHy0wy
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0qAt19mQfz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HbeMeIFKef
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0qAt19mQfz
++ cat /tmp/tmp.HbeMeIFKef
++ rm /tmp/tmp.0qAt19mQfz /tmp/tmp.HbeMeIFKef
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3A9I5alkGz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0SJ0WWCVCX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3A9I5alkGz
++ cat /tmp/tmp.0SJ0WWCVCX
++ rm /tmp/tmp.3A9I5alkGz /tmp/tmp.0SJ0WWCVCX
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish.....OK
+ [[ true == true ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zkuT7X92m1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TBtJUi8Bv8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zkuT7X92m1
++ cat /tmp/tmp.TBtJUi8Bv8
++ rm /tmp/tmp.zkuT7X92m1 /tmp/tmp.TBtJUi8Bv8
++ return 0
+ local client_container=psmdb-client-b9788d8bc-jfqzw
+ kubectl_bin exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rXECEB8PEU
++ mktemp
+ local LAST_ERR=/tmp/tmp.nB4oBNgn4m
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-jfqzw -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-19753.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rXECEB8PEU
+ cat /tmp/tmp.nB4oBNgn4m
+ rm /tmp/tmp.rXECEB8PEU /tmp/tmp.nB4oBNgn4m
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.3UJ4bnYUHf/find-sharded
+ destroy demand-backup-incremental-sharded-19753
+ local namespace=demand-backup-incremental-sharded-19753
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BG8qTIZode
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PeRRguDWh8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BG8qTIZode
++ cat /tmp/tmp.PeRRguDWh8
++ rm /tmp/tmp.BG8qTIZode /tmp/tmp.PeRRguDWh8
++ return 0
+ '[' 5 '!=' 0 ']'
+ kubectl_bin get psmdb-backup
++ mktemp
+ local LAST_OUT=/tmp/tmp.Y7SEJao43y
++ mktemp
+ local LAST_ERR=/tmp/tmp.mL87fyrjNc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb-backup
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Y7SEJao43y
NAME                            CLUSTER     STORAGE      DESTINATION                                                                             TYPE               SIZE     STATUS   COMPLETED   AGE
backup-aws-s3-sharded           some-name   aws-s3       s3://operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:02:26Z      incremental-base   1.78MB   ready    52m         53m
backup-azure-blob-sharded       some-name   azure-blob   azure://operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:42Z   incremental-base   2.45MB   ready    51m         53m
backup-gcp-cs-sharded           some-name   gcp-cs       s3://operator-testing/psmdb-demand-backup-incremental-sharded/2025-10-09T19:03:09Z      incremental-base   2.51MB   ready    52m         53m
backup-minio-not-base-sharded   some-name   minio        s3://operator-testing/2025-10-09T19:04:53Z                                              incremental        2.94MB   ready    50m         50m
backup-minio-sharded            some-name   minio        s3://operator-testing/2025-10-09T19:04:19Z                                              incremental-base   3.45MB   ready    51m         51m
+ cat /tmp/tmp.mL87fyrjNc
+ rm /tmp/tmp.Y7SEJao43y /tmp/tmp.mL87fyrjNc
+ return 0
+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.oWes7jv49a
++ mktemp
+ local LAST_ERR=/tmp/tmp.J77duhSXcI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oWes7jv49a
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3-sharded" deleted from demand-backup-incremental-sharded-19753 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob-sharded" deleted from demand-backup-incremental-sharded-19753 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sharded" deleted from demand-backup-incremental-sharded-19753 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-not-base-sharded" deleted from demand-backup-incremental-sharded-19753 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio-sharded" deleted from demand-backup-incremental-sharded-19753 namespace
+ cat /tmp/tmp.J77duhSXcI
+ rm /tmp/tmp.oWes7jv49a /tmp/tmp.J77duhSXcI
+ return 0
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.RORzwOzoKw
++ mktemp
+ local LAST_ERR=/tmp/tmp.TwGF0NNKfV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RORzwOzoKw
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.TwGF0NNKfV
+ rm /tmp/tmp.RORzwOzoKw /tmp/tmp.TwGF0NNKfV
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.fUeDSZUd3v
++ mktemp
+ local LAST_ERR=/tmp/tmp.Yn7aLyq8rm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fUeDSZUd3v
+ cat /tmp/tmp.Yn7aLyq8rm
+ rm /tmp/tmp.fUeDSZUd3v /tmp/tmp.Yn7aLyq8rm
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.CY3Fdi97Ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.dCwi2grbEv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CY3Fdi97Ns
+ cat /tmp/tmp.dCwi2grbEv
+ rm /tmp/tmp.CY3Fdi97Ns /tmp/tmp.dCwi2grbEv
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.1PXKDvVHm7
++ mktemp
+ local LAST_ERR=/tmp/tmp.u6RV4GLD62
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1PXKDvVHm7
+ cat /tmp/tmp.u6RV4GLD62
+ rm /tmp/tmp.1PXKDvVHm7 /tmp/tmp.u6RV4GLD62
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.YTPCWMKEPU
++ mktemp
+ local LAST_ERR=/tmp/tmp.5XBc1UoABO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2083/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YTPCWMKEPU
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.5XBc1UoABO
+ rm /tmp/tmp.YTPCWMKEPU /tmp/tmp.5XBc1UoABO
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.mJCFOGjdTc
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZFCivT9kYS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.mJCFOGjdTc
+ cat /tmp/tmp.ZFCivT9kYS
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.mJCFOGjdTc
+ cat /tmp/tmp.ZFCivT9kYS
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.mJCFOGjdTc
+ cat /tmp/tmp.ZFCivT9kYS
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.mJCFOGjdTc
+ cat /tmp/tmp.ZFCivT9kYS
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.mJCFOGjdTc /tmp/tmp.ZFCivT9kYS
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-incremental-sharded-19753
+ rm -rf /tmp/tmp.3UJ4bnYUHf
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.D6AZMZUMbo
+ local LAST_OUT=/tmp/tmp.AbuplYTdyv
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.i8BFlI4ZJa
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.2W0u0v3m6t
++ seq 0 2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-incremental-sharded-19753