++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/logs/demand-backup-sharded.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/logs/demand-backup-sharded.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ kubectl get nodes
++ grep '^minikube'
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep -eks-
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
++ '[' ']'
++ EKS=0
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep gke
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
++ '[' v1.33.11-gke.1074000 ']'
++ GKE=1
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ /usr/sbin/sed -r 's/[^0-9.]+//g'
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.33
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ custom_port=27019
+ create_infra demand-backup-sharded-429
+ local ns=demand-backup-sharded-429
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.XktNAdzyYO
++ mktemp
+ local LAST_ERR=/tmp/tmp.jv5gnyxkGH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XktNAdzyYO
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.jv5gnyxkGH
+ rm /tmp/tmp.XktNAdzyYO /tmp/tmp.jv5gnyxkGH
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-sharded-6706 backup-aws-s3 --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-sharded-6706 backup-azure-blob --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-sharded-6706 backup-gcp-cs --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-sharded-6706 backup-minio --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.FHiwTpGASJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.uKRYNKvgzm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FHiwTpGASJ
+ cat /tmp/tmp.uKRYNKvgzm
+ rm /tmp/tmp.FHiwTpGASJ /tmp/tmp.uKRYNKvgzm
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.EyP5KJ3zZW
++ mktemp
+ local LAST_ERR=/tmp/tmp.7P5DYsBk3l
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EyP5KJ3zZW
+ cat /tmp/tmp.7P5DYsBk3l
+ rm /tmp/tmp.EyP5KJ3zZW /tmp/tmp.7P5DYsBk3l
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.HZ1awrDAlV
++ mktemp
+ local LAST_ERR=/tmp/tmp.pTWq26ofIn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HZ1awrDAlV
+ cat /tmp/tmp.pTWq26ofIn
+ rm /tmp/tmp.HZ1awrDAlV /tmp/tmp.pTWq26ofIn
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.LgomGlIR6W
++ mktemp
+ local LAST_ERR=/tmp/tmp.bWvP3tfCgU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LgomGlIR6W
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.bWvP3tfCgU
+ rm /tmp/tmp.LgomGlIR6W /tmp/tmp.bWvP3tfCgU
+ return 0
+ check_crd_for_deletion PR-2381-c451228ca
+ local git_tag=PR-2381-c451228ca
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2381-c451228ca/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RbRwILocWz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O0UzCuEdtV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.RbRwILocWz
++ cat /tmp/tmp.O0UzCuEdtV
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.RbRwILocWz
++ cat /tmp/tmp.O0UzCuEdtV
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.RbRwILocWz
++ cat /tmp/tmp.O0UzCuEdtV
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.RbRwILocWz
++ cat /tmp/tmp.O0UzCuEdtV
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.RbRwILocWz /tmp/tmp.O0UzCuEdtV
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ awk '{print$1}'
+ xargs kubectl delete ns
+ local LAST_OUT=/tmp/tmp.TjO6oKbROv
++ mktemp
+ local LAST_OUT=/tmp/tmp.smvhPFSZN1
++ mktemp
+ local LAST_ERR=/tmp/tmp.XLKFKA0gRv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.RQDRU2tRqP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TjO6oKbROv
+ cat /tmp/tmp.XLKFKA0gRv
+ rm /tmp/tmp.TjO6oKbROv /tmp/tmp.XLKFKA0gRv
+ return 0
namespace "demand-backup-sharded-6706" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.smvhPFSZN1
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.RQDRU2tRqP
+ rm /tmp/tmp.smvhPFSZN1 /tmp/tmp.RQDRU2tRqP
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.DTf2oV1vp2
++ mktemp
+ local LAST_ERR=/tmp/tmp.O69YeejN2y
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DTf2oV1vp2
+ cat /tmp/tmp.O69YeejN2y
+ rm /tmp/tmp.DTf2oV1vp2 /tmp/tmp.O69YeejN2y
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.XYeB5Ta2ht
++ mktemp
+ local LAST_ERR=/tmp/tmp.vlLzMcPqHn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XYeB5Ta2ht
namespace/psmdb-operator created
+ cat /tmp/tmp.vlLzMcPqHn
+ rm /tmp/tmp.XYeB5Ta2ht /tmp/tmp.vlLzMcPqHn
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PGgn7pb04S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IV40Cfe7Gy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PGgn7pb04S
++ cat /tmp/tmp.IV40Cfe7Gy
++ rm /tmp/tmp.PGgn7pb04S /tmp/tmp.IV40Cfe7Gy
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2381-c451228ca-12-cluster1 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.Mh5ndZUWMb
++ mktemp
+ local LAST_ERR=/tmp/tmp.RpEz8EvmG0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2381-c451228ca-12-cluster1 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Mh5ndZUWMb
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2381-c451228ca-12-cluster1" modified.
+ cat /tmp/tmp.RpEz8EvmG0
+ rm /tmp/tmp.Mh5ndZUWMb /tmp/tmp.RpEz8EvmG0
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2381-c451228ca'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2381-c451228ca
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.xbxiMu07fa
++ mktemp
+ local LAST_ERR=/tmp/tmp.sgFIjM1tlY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xbxiMu07fa
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.sgFIjM1tlY
+ rm /tmp/tmp.xbxiMu07fa /tmp/tmp.sgFIjM1tlY
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.vyig5p4SfS
++ mktemp
+ local LAST_ERR=/tmp/tmp.PX92WKEjw8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vyig5p4SfS
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.PX92WKEjw8
+ rm /tmp/tmp.vyig5p4SfS /tmp/tmp.PX92WKEjw8
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2381-c451228ca") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.pemZrtqnmz
++ mktemp
+ local LAST_ERR=/tmp/tmp.oHPqrAwD4c
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pemZrtqnmz
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.oHPqrAwD4c
+ rm /tmp/tmp.pemZrtqnmz /tmp/tmp.oHPqrAwD4c
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qpLXkX4oer
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PsqNjjRPG0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qpLXkX4oer
++ cat /tmp/tmp.PsqNjjRPG0
++ rm /tmp/tmp.qpLXkX4oer /tmp/tmp.PsqNjjRPG0
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-5b57d4786b-tkl4b
+ local pod=percona-server-mongodb-operator-5b57d4786b-tkl4b
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-5b57d4786b-tkl4b to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sFsYdcPx9B
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DHYQkAncXC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sFsYdcPx9B
++ cat /tmp/tmp.DHYQkAncXC
++ rm /tmp/tmp.sFsYdcPx9B /tmp/tmp.DHYQkAncXC
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-5b57d4786b-tkl4b
++ mktemp
+ local LAST_OUT=/tmp/tmp.024CXHP0Oo
++ mktemp
+ local LAST_ERR=/tmp/tmp.VnUyvWypyp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-5b57d4786b-tkl4b
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.024CXHP0Oo
+ cat /tmp/tmp.VnUyvWypyp
+ rm /tmp/tmp.024CXHP0Oo /tmp/tmp.VnUyvWypyp
+ return 0
2026-06-11T12:18:35.883Z	INFO	setup	Manager starting up	{"gitCommit": "c451228caa4da9929b85b557dcf90547304a2519", "gitBranch": "PR-2381-c451228ca", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-sharded-429
+ local namespace=demand-backup-sharded-429
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-sharded-429'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-sharded-429
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-sharded-429 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.N8yh5iiHwy
++ mktemp
+ local LAST_OUT=/tmp/tmp.eNhzMpH3qt
++ mktemp
+ local LAST_ERR=/tmp/tmp.fhzxwYDlzv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.Q33JlsEatx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-sharded-429 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N8yh5iiHwy
+ cat /tmp/tmp.fhzxwYDlzv
+ rm /tmp/tmp.N8yh5iiHwy /tmp/tmp.fhzxwYDlzv
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eNhzMpH3qt
+ cat /tmp/tmp.Q33JlsEatx
+ rm /tmp/tmp.eNhzMpH3qt /tmp/tmp.Q33JlsEatx
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-sharded-429
++ mktemp
+ local LAST_OUT=/tmp/tmp.dfaXGHV3XR
++ mktemp
error: resource(s) were provided, but no name was specified
+ local LAST_ERR=/tmp/tmp.CykElETQnb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-sharded-429
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dfaXGHV3XR
+ cat /tmp/tmp.CykElETQnb
+ rm /tmp/tmp.dfaXGHV3XR /tmp/tmp.CykElETQnb
+ return 0
+ desc 'create namespace demand-backup-sharded-429'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-sharded-429
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-sharded-429
++ mktemp
+ local LAST_OUT=/tmp/tmp.0cU42PVnpd
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZkB9eKzcJI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-sharded-429
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0cU42PVnpd
namespace/demand-backup-sharded-429 created
+ cat /tmp/tmp.ZkB9eKzcJI
+ rm /tmp/tmp.0cU42PVnpd /tmp/tmp.ZkB9eKzcJI
+ return 0
+ set_kube_ctx demand-backup-sharded-429
+ local namespace=demand-backup-sharded-429
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ORo9BzOUfr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZxrqswxrIN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ORo9BzOUfr
++ cat /tmp/tmp.ZxrqswxrIN
++ rm /tmp/tmp.ORo9BzOUfr /tmp/tmp.ZxrqswxrIN
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2381-c451228ca-12-cluster1 --namespace=demand-backup-sharded-429
++ mktemp
+ local LAST_OUT=/tmp/tmp.PCjL9xMVOI
++ mktemp
+ local LAST_ERR=/tmp/tmp.owMhh83iyW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2381-c451228ca-12-cluster1 --namespace=demand-backup-sharded-429
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PCjL9xMVOI
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2381-c451228ca-12-cluster1" modified.
+ cat /tmp/tmp.owMhh83iyW
+ rm /tmp/tmp.PCjL9xMVOI /tmp/tmp.owMhh83iyW
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Thu Jun 11 12:19:13 2026
NAMESPACE: demand-backup-sharded-429
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-sharded-429.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-429 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-429

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-sharded-429 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-sharded-429 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3uVmVFgMBK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tNhQgyFqMe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3uVmVFgMBK
++ cat /tmp/tmp.tNhQgyFqMe
++ rm /tmp/tmp.3uVmVFgMBK /tmp/tmp.tNhQgyFqMe
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-7fgtp
+ wait_pod minio-service-6d5f646cdc-7fgtp
+ local pod=minio-service-6d5f646cdc-7fgtp
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-7fgtp to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-sharded-429.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.fZqJWAHI1O
++ mktemp
+ local LAST_ERR=/tmp/tmp.cSPjrQ90Ws
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fZqJWAHI1O
make_bucket: operator-testing
pod "aws-cli" deleted from demand-backup-sharded-429 namespace
+ cat /tmp/tmp.cSPjrQ90Ws
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.fZqJWAHI1O /tmp/tmp.cSPjrQ90Ws
+ return 0
+ desc 'create first PSMDB cluster with custom port 27019'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster with custom port 27019
-----------------------------------------------------------------------------------

+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.IvmtPa1rwa
++ mktemp
+ local LAST_ERR=/tmp/tmp.4vHePlEsQL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IvmtPa1rwa
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.4vHePlEsQL
+ rm /tmp/tmp.IvmtPa1rwa /tmp/tmp.4vHePlEsQL
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.bgWjw20aEM
++ mktemp
+ local LAST_ERR=/tmp/tmp.so1A6y1nd2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bgWjw20aEM
secret/minio-secret created
secret/aws-s3-secret created
secret/alibaba-cloud-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.so1A6y1nd2
+ rm /tmp/tmp.bgWjw20aEM /tmp/tmp.so1A6y1nd2
+ return 0
+ version_gt 1.19
++ echo '1.33 >= 1.19'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' 0 -ne 1 ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/conf/container-rc.yaml
+ /usr/sbin/sed s/docker/runc/g
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.J5uap1E0oH
++ mktemp
+ local LAST_ERR=/tmp/tmp.Mp22gvNUsL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J5uap1E0oH
runtimeclass.node.k8s.io/container-rc unchanged
+ cat /tmp/tmp.Mp22gvNUsL
+ rm /tmp/tmp.J5uap1E0oH /tmp/tmp.Mp22gvNUsL
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ /usr/sbin/sed -e s/NAME_SPACE/demand-backup-sharded-429/g
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2381-c451228ca"'
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ local LAST_OUT=/tmp/tmp.hOoqceTFX9
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.plY2RSarcf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hOoqceTFX9
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.plY2RSarcf
+ rm /tmp/tmp.hOoqceTFX9 /tmp/tmp.plY2RSarcf
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tea4vrwa7I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GZw92Xc8Bw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tea4vrwa7I
++ cat /tmp/tmp.GZw92Xc8Bw
++ rm /tmp/tmp.tea4vrwa7I /tmp/tmp.GZw92Xc8Bw
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5nmO7GZ59y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gKPFKo5Vqb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5nmO7GZ59y
++ cat /tmp/tmp.gKPFKo5Vqb
++ rm /tmp/tmp.5nmO7GZ59y /tmp/tmp.gKPFKo5Vqb
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nCMqLkJRaR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UScG04fIUb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nCMqLkJRaR
++ cat /tmp/tmp.UScG04fIUb
++ rm /tmp/tmp.nCMqLkJRaR /tmp/tmp.UScG04fIUb
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness.........................................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YctROJlWth
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ylkKkLEuwe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YctROJlWth
++ cat /tmp/tmp.ylkKkLEuwe
++ rm /tmp/tmp.YctROJlWth /tmp/tmp.ylkKkLEuwe
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aZpX00W1kL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xyZANwO67r
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aZpX00W1kL
++ cat /tmp/tmp.xyZANwO67r
++ rm /tmp/tmp.aZpX00W1kL /tmp/tmp.xyZANwO67r
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KXdJjz3R2W
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BXRNUc1nBf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KXdJjz3R2W
++ cat /tmp/tmp.BXRNUc1nBf
++ rm /tmp/tmp.KXdJjz3R2W /tmp/tmp.BXRNUc1nBf
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ false == true ]]
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VbspEU2lja
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jFBUyP2FNL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VbspEU2lja
++ cat /tmp/tmp.jFBUyP2FNL
++ rm /tmp/tmp.VbspEU2lja /tmp/tmp.jFBUyP2FNL
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2wy36XfRIT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.suxrh7d4Gb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2wy36XfRIT
++ cat /tmp/tmp.suxrh7d4Gb
++ rm /tmp/tmp.2wy36XfRIT /tmp/tmp.suxrh7d4Gb
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E0tnt2YmrC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4nFCLUPSiv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E0tnt2YmrC
++ cat /tmp/tmp.4nFCLUPSiv
++ rm /tmp/tmp.E0tnt2YmrC /tmp/tmp.4nFCLUPSiv
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-429", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.6hgxXDsh84
++ mktemp
+ local LAST_ERR=/tmp/tmp.bmy6HkOu0G
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6hgxXDsh84
+ cat /tmp/tmp.bmy6HkOu0G
+ rm /tmp/tmp.6hgxXDsh84 /tmp/tmp.bmy6HkOu0G
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml /tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-06-11T12:24:15+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ compare_kubectl statefulset/some-name-rs1
+ local resource=statefulset/some-name-rs1
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml
+ local new_result=/tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs1.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs1
++ mktemp
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-429", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.6VOzw4On5I
++ mktemp
+ local LAST_ERR=/tmp/tmp.TXJnm67zix
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6VOzw4On5I
+ cat /tmp/tmp.TXJnm67zix
+ rm /tmp/tmp.6VOzw4On5I /tmp/tmp.TXJnm67zix
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs1.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs1.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs1.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml /tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs1.yml
+ log 'compare_kubectl: statefulset/some-name-rs1 OK'
+ set +o xtrace
[2026-06-11T12:24:16+0000] compare_kubectl: statefulset/some-name-rs1 OK
+ compare_kubectl statefulset/some-name-rs2
+ local resource=statefulset/some-name-rs2
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml
+ local new_result=/tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs2.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs2
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-429", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ROFxObrpZP
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZxlMbbflEZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs2
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ROFxObrpZP
+ cat /tmp/tmp.ZxlMbbflEZ
+ rm /tmp/tmp.ROFxObrpZP /tmp/tmp.ZxlMbbflEZ
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs2.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs2.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs2.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml /tmp/tmp.KZJaOOCJa7/statefulset_some-name-rs2.yml
+ log 'compare_kubectl: statefulset/some-name-rs2 OK'
+ set +o xtrace
[2026-06-11T12:24:17+0000] compare_kubectl: statefulset/some-name-rs2 OK
+ compare_kubectl statefulset/some-name-cfg
+ local resource=statefulset/some-name-cfg
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml
+ local new_result=/tmp/tmp.KZJaOOCJa7/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-429", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.YIEN5h4w7h
++ mktemp
+ local LAST_ERR=/tmp/tmp.5UxIH9GQqz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YIEN5h4w7h
+ cat /tmp/tmp.5UxIH9GQqz
+ rm /tmp/tmp.YIEN5h4w7h /tmp/tmp.5UxIH9GQqz
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml /tmp/tmp.KZJaOOCJa7/statefulset_some-name-cfg.yml
+ log 'compare_kubectl: statefulset/some-name-cfg OK'
+ set +o xtrace
[2026-06-11T12:24:18+0000] compare_kubectl: statefulset/some-name-cfg OK
+ compare_kubectl statefulset/some-name-mongos ''
+ local resource=statefulset/some-name-mongos
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml
+ local new_result=/tmp/tmp.KZJaOOCJa7/statefulset_some-name-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-mongos
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-429", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.03GlEZ5tch
++ mktemp
+ local LAST_ERR=/tmp/tmp.oZl0spZGgV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.03GlEZ5tch
+ cat /tmp/tmp.oZl0spZGgV
+ rm /tmp/tmp.03GlEZ5tch /tmp/tmp.oZl0spZGgV
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-mongos.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml /tmp/tmp.KZJaOOCJa7/statefulset_some-name-mongos.yml
+ log 'compare_kubectl: statefulset/some-name-mongos OK'
+ set +o xtrace
[2026-06-11T12:24:19+0000] compare_kubectl: statefulset/some-name-mongos OK
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.qGZlVemwvl
++ mktemp
+ local LAST_ERR=/tmp/tmp.weDAKzOD4S
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qGZlVemwvl
secret/some-name-mongos created
+ cat /tmp/tmp.weDAKzOD4S
+ rm /tmp/tmp.qGZlVemwvl /tmp/tmp.weDAKzOD4S
+ return 0
+ sleep 10
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XBDC9dgCaG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B6tzstdBN7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XBDC9dgCaG
++ cat /tmp/tmp.B6tzstdBN7
++ rm /tmp/tmp.XBDC9dgCaG /tmp/tmp.B6tzstdBN7
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.metwSxxw82
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iOQ35hrac6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.metwSxxw82
++ cat /tmp/tmp.iOQ35hrac6
++ rm /tmp/tmp.metwSxxw82 /tmp/tmp.iOQ35hrac6
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VYfDThB4Vl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XK04y0DkW9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VYfDThB4Vl
++ cat /tmp/tmp.XK04y0DkW9
++ rm /tmp/tmp.VYfDThB4Vl /tmp/tmp.XK04y0DkW9
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness...................................................................................................
+ compare_kubectl statefulset/some-name-mongos -secret
+ local resource=statefulset/some-name-mongos
+ local postfix=-secret
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml
+ local new_result=/tmp/tmp.KZJaOOCJa7/statefulset_some-name-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-mongos
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-429", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ctvT0Ozf1c
++ mktemp
+ local LAST_ERR=/tmp/tmp.NIBkMB1tTC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ctvT0Ozf1c
+ cat /tmp/tmp.NIBkMB1tTC
+ rm /tmp/tmp.ctvT0Ozf1c /tmp/tmp.NIBkMB1tTC
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-mongos.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.KZJaOOCJa7/statefulset_some-name-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml /tmp/tmp.KZJaOOCJa7/statefulset_some-name-mongos.yml
+ log 'compare_kubectl: statefulset/some-name-mongos OK'
+ set +o xtrace
[2026-06-11T12:28:22+0000] compare_kubectl: statefulset/some-name-mongos OK
+ desc 'checking hookscripts'
+ set +o xtrace


-----------------------------------------------------------------------------------
checking hookscripts
-----------------------------------------------------------------------------------

+ kubectl_bin logs some-name-rs0-0 backup-agent
+ grep -q 'Backup Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.YfL99XeX8r
++ mktemp
+ local LAST_ERR=/tmp/tmp.e7B5Qw8kWL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-rs0-0 backup-agent
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YfL99XeX8r
+ cat /tmp/tmp.e7B5Qw8kWL
+ rm /tmp/tmp.YfL99XeX8r /tmp/tmp.e7B5Qw8kWL
+ return 0
+ kubectl_bin logs some-name-rs0-0
+ grep -q 'RS0 Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.aZlx16CR1i
++ mktemp
+ local LAST_ERR=/tmp/tmp.R2oi3mPdSC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-rs0-0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aZlx16CR1i
cat: write error: Broken pipe
+ cat /tmp/tmp.R2oi3mPdSC
Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init)
+ rm /tmp/tmp.aZlx16CR1i /tmp/tmp.R2oi3mPdSC
+ return 0
+ kubectl_bin logs some-name-rs1-0 backup-agent
+ grep -q 'Backup Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.S2b1ISwYTn
++ mktemp
+ local LAST_ERR=/tmp/tmp.efh4k4Freb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-rs1-0 backup-agent
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.S2b1ISwYTn
+ cat /tmp/tmp.efh4k4Freb
+ rm /tmp/tmp.S2b1ISwYTn /tmp/tmp.efh4k4Freb
+ return 0
+ kubectl_bin logs some-name-rs1-0
+ grep -q 'RS1 Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.NGjkBjcG0F
++ mktemp
+ local LAST_ERR=/tmp/tmp.Jeu6NRYmTv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-rs1-0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NGjkBjcG0F
cat: write error: Broken pipe
+ cat /tmp/tmp.Jeu6NRYmTv
Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init)
+ rm /tmp/tmp.NGjkBjcG0F /tmp/tmp.Jeu6NRYmTv
+ return 0
+ kubectl_bin logs some-name-rs2-0 backup-agent
+ grep -q 'Backup Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZsyQ3Wzyfn
++ mktemp
+ local LAST_ERR=/tmp/tmp.xzWRdrgA7V
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-rs2-0 backup-agent
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZsyQ3Wzyfn
+ cat /tmp/tmp.xzWRdrgA7V
+ rm /tmp/tmp.ZsyQ3Wzyfn /tmp/tmp.xzWRdrgA7V
+ return 0
+ kubectl_bin logs some-name-rs2-0
+ grep -q 'RS2 Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.qoKS1n2VXq
++ mktemp
+ local LAST_ERR=/tmp/tmp.nRT0xJeURL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-rs2-0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qoKS1n2VXq
cat: write error: Broken pipe
+ cat /tmp/tmp.nRT0xJeURL
Defaulted container "mongod" out of: mongod, rs-sidecar-1, backup-agent, mongo-init (init)
+ rm /tmp/tmp.qoKS1n2VXq /tmp/tmp.nRT0xJeURL
+ return 0
+ kubectl_bin logs some-name-cfg-0 backup-agent
+ grep -q 'Backup Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.PcCIDXedrq
++ mktemp
+ local LAST_ERR=/tmp/tmp.aa8Ai2BLMF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-cfg-0 backup-agent
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PcCIDXedrq
+ cat /tmp/tmp.aa8Ai2BLMF
+ rm /tmp/tmp.PcCIDXedrq /tmp/tmp.aa8Ai2BLMF
+ return 0
+ kubectl_bin logs some-name-cfg-0
+ grep -q 'CFG Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.EFbyz94wfw
++ mktemp
+ local LAST_ERR=/tmp/tmp.99Z9sfP67T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-cfg-0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EFbyz94wfw
cat: write error: Broken pipe
+ cat /tmp/tmp.99Z9sfP67T
Defaulted container "mongod" out of: mongod, cfg-sidecar-1, backup-agent, mongo-init (init)
+ rm /tmp/tmp.EFbyz94wfw /tmp/tmp.99Z9sfP67T
+ return 0
+ kubectl_bin logs some-name-mongos-0
+ grep -q 'Mongos Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.F7npAWUV3k
++ mktemp
+ local LAST_ERR=/tmp/tmp.DAOPaJXLbH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-mongos-0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.F7npAWUV3k
cat: write error: Broken pipe
+ cat /tmp/tmp.DAOPaJXLbH
Defaulted container "mongos" out of: mongos, mongos-sidecar-1, mongo-init (init)
+ rm /tmp/tmp.F7npAWUV3k /tmp/tmp.DAOPaJXLbH
+ return 0
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7Z6695jBea
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iBZxn0Cav4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7Z6695jBea
++ cat /tmp/tmp.iBZxn0Cav4
++ rm /tmp/tmp.7Z6695jBea /tmp/tmp.iBZxn0Cav4
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.J7fZS4p8lE
++ mktemp
+ local LAST_ERR=/tmp/tmp.ed8RcCS27p
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J7fZS4p8lE
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("827e3def-44f8-4139-ba4d-d46664a81664") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		},
		{
			"db" : "myApp1",
			"role" : "readWrite"
		},
		{
			"db" : "myApp2",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.ed8RcCS27p
+ rm /tmp/tmp.J7fZS4p8lE /tmp/tmp.ed8RcCS27p
+ return 0
+ run_mongos 'sh.enableSharding("myApp","rs0")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=sh.enableSharding("myApp","rs0")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3Gd1hwg4SK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.p1W8arJ0JL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3Gd1hwg4SK
++ cat /tmp/tmp.p1W8arJ0JL
++ rm /tmp/tmp.3Gd1hwg4SK /tmp/tmp.p1W8arJ0JL
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mKFJ8AHRRs
++ mktemp
+ local LAST_ERR=/tmp/tmp.6L2bxRtEpY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mKFJ8AHRRs
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("818c743d-c2d8-4d52-b044-5f3b310f04dd") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1781180921, 8),
		"signature" : {
			"hash" : BinData(0,"C3cAqI3t+v76MK4MXES420a7iYU="),
			"keyId" : NumberLong("7650112180456521752")
		}
	},
	"operationTime" : Timestamp(1781180921, 5)
}
bye
+ cat /tmp/tmp.6L2bxRtEpY
+ rm /tmp/tmp.mKFJ8AHRRs /tmp/tmp.6L2bxRtEpY
+ return 0
+ run_mongos 'sh.enableSharding("myApp1","rs1")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=sh.enableSharding("myApp1","rs1")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GSsh7pTP4L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PqphJt8GYe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GSsh7pTP4L
++ cat /tmp/tmp.PqphJt8GYe
++ rm /tmp/tmp.GSsh7pTP4L /tmp/tmp.PqphJt8GYe
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FE2e5iPf6W
++ mktemp
+ local LAST_ERR=/tmp/tmp.pfEZI1o9KX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FE2e5iPf6W
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("0d0f8f5b-dc0c-4962-9727-d3563ade6e5d") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1781180924, 9),
		"signature" : {
			"hash" : BinData(0,"Pu1gUxfCq4q+oKw3/ABQ9CA9lCI="),
			"keyId" : NumberLong("7650112180456521752")
		}
	},
	"operationTime" : Timestamp(1781180924, 6)
}
bye
+ cat /tmp/tmp.pfEZI1o9KX
+ rm /tmp/tmp.FE2e5iPf6W /tmp/tmp.pfEZI1o9KX
+ return 0
+ run_mongos 'sh.enableSharding("myApp2","rs2")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=sh.enableSharding("myApp2","rs2")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WrsTrAeOUe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Zn99ljYqUt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WrsTrAeOUe
++ cat /tmp/tmp.Zn99ljYqUt
++ rm /tmp/tmp.WrsTrAeOUe /tmp/tmp.Zn99ljYqUt
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.P8XCf3NDwi
++ mktemp
+ local LAST_ERR=/tmp/tmp.moySnq5mIM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.P8XCf3NDwi
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("4423d8ab-de93-4238-8e2f-fe156a796df9") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1781180926, 8),
		"signature" : {
			"hash" : BinData(0,"iIAtO6T9f7Xx3SXPdM02ETifumM="),
			"keyId" : NumberLong("7650112180456521752")
		}
	},
	"operationTime" : Timestamp(1781180926, 5)
}
bye
+ cat /tmp/tmp.moySnq5mIM
+ rm /tmp/tmp.P8XCf3NDwi /tmp/tmp.moySnq5mIM
+ return 0
+ insert_data_mongos 100500 myApp '' 27019
+ local data=100500
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jSSS7UUDWX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WYZiS9hFRF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jSSS7UUDWX
++ cat /tmp/tmp.WYZiS9hFRF
++ rm /tmp/tmp.jSSS7UUDWX /tmp/tmp.WYZiS9hFRF
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7m16FNLNDS
++ mktemp
+ local LAST_ERR=/tmp/tmp.7CYxe9G6sr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7m16FNLNDS
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b3c3204f-b75f-498c-8b04-1f3bf127bbbc") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.7CYxe9G6sr
+ rm /tmp/tmp.7m16FNLNDS /tmp/tmp.7CYxe9G6sr
+ return 0
+ insert_data_mongos 100500 myApp1 '' 27019
+ local data=100500
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.02l10952a4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.p14HMXjHEP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.02l10952a4
++ cat /tmp/tmp.p14HMXjHEP
++ rm /tmp/tmp.02l10952a4 /tmp/tmp.p14HMXjHEP
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DCeLVyHgl7
++ mktemp
+ local LAST_ERR=/tmp/tmp.7OHRQj57NH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DCeLVyHgl7
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("ed759725-4e78-4be6-b888-f539b7a0a9e9") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.7OHRQj57NH
+ rm /tmp/tmp.DCeLVyHgl7 /tmp/tmp.7OHRQj57NH
+ return 0
+ insert_data_mongos 100500 myApp2 '' 27019
+ local data=100500
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N0KE4LtLZu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CFEDqkDayO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.N0KE4LtLZu
++ cat /tmp/tmp.CFEDqkDayO
++ rm /tmp/tmp.N0KE4LtLZu /tmp/tmp.CFEDqkDayO
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.6lucBdKrU0
++ mktemp
+ local LAST_ERR=/tmp/tmp.Qmx7dgmclj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6lucBdKrU0
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e8a7ef03-9a1b-4874-9523-ed2cc981cfea") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Qmx7dgmclj
+ rm /tmp/tmp.6lucBdKrU0 /tmp/tmp.Qmx7dgmclj
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' '' 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-11T12:28:53+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb '' '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.j2gghOwrVc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vfKz37COME
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.j2gghOwrVc
++ cat /tmp/tmp.vfKz37COME
++ rm /tmp/tmp.j2gghOwrVc /tmp/tmp.vfKz37COME
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.f9sAI3f68m
++ mktemp
+ local LAST_ERR=/tmp/tmp.R2pxOvZQ2y
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.f9sAI3f68m
+ cat /tmp/tmp.R2pxOvZQ2y
+ rm /tmp/tmp.f9sAI3f68m /tmp/tmp.R2pxOvZQ2y
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.KZJaOOCJa7/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2026-06-11T12:23:04.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2026-06-11T12:23:07.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2026-06-11T12:23:24.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-0
+ local agent_pod=some-name-rs1-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-0...2026-06-11T12:23:05.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-1
+ local agent_pod=some-name-rs1-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-1...2026-06-11T12:23:08.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-2
+ local agent_pod=some-name-rs1-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-2...2026-06-11T12:23:25.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-0
+ local agent_pod=some-name-rs2-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-0...2026-06-11T12:23:05.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-1
+ local agent_pod=some-name-rs2-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-1...2026-06-11T12:23:09.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-2
+ local agent_pod=some-name-rs2-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-2...2026-06-11T12:23:26.000+0000 I listening for the commands

+ backup_name_aws=backup-aws-s3
+ backup_name_minio=backup-minio
+ backup_name_gcp=backup-gcp-cs
+ backup_name_azure=backup-azure-blob
+ desc 'run backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------

+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ log 'running backup backup-minio'
+ set +o xtrace
[2026-06-11T12:29:08+0000] running backup backup-minio
+ yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.2xwHXUJEyx
++ mktemp
+ local LAST_ERR=/tmp/tmp.WSpdOyio0u
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2xwHXUJEyx
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.WSpdOyio0u
+ rm /tmp/tmp.2xwHXUJEyx /tmp/tmp.WSpdOyio0u
+ return 0
+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ log 'running backup backup-aws-s3'
+ set +o xtrace
[2026-06-11T12:29:09+0000] running backup backup-aws-s3
+ yq eval $'.metadata.name = "backup-aws-s3"\n\t\t\t| .spec.storageName = "aws-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.YaIEgRIZZC
++ mktemp
+ local LAST_ERR=/tmp/tmp.B6LNv4OcmW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YaIEgRIZZC
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.B6LNv4OcmW
+ rm /tmp/tmp.YaIEgRIZZC /tmp/tmp.B6LNv4OcmW
+ return 0
+ run_backup gcp-cs
+ local storage=gcp-cs
+ local backup_name=backup-gcp-cs
+ local type=logical
+ log 'running backup backup-gcp-cs'
+ set +o xtrace
[2026-06-11T12:29:12+0000] running backup backup-gcp-cs
+ yq eval $'.metadata.name = "backup-gcp-cs"\n\t\t\t| .spec.storageName = "gcp-cs"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/backup-gcp-cs.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.NYU24tsUJO
++ mktemp
+ local LAST_ERR=/tmp/tmp.RWn83Rm84w
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NYU24tsUJO
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created
+ cat /tmp/tmp.RWn83Rm84w
+ rm /tmp/tmp.NYU24tsUJO /tmp/tmp.RWn83Rm84w
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ log 'running backup backup-azure-blob'
+ set +o xtrace
[2026-06-11T12:29:14+0000] running backup backup-azure-blob
+ kubectl_bin apply -f -
+ yq eval $'.metadata.name = "backup-azure-blob"\n\t\t\t| .spec.storageName = "azure-blob"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/backup-azure-blob.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZvXZukHfaq
++ mktemp
+ local LAST_ERR=/tmp/tmp.TVeHkvkioG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZvXZukHfaq
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.TVeHkvkioG
+ rm /tmp/tmp.ZvXZukHfaq /tmp/tmp.TVeHkvkioG
+ return 0
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state...............................OK
+ wait_backup backup-gcp-cs
+ local backup_name=backup-gcp-cs
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs to reach ready state..................OK
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state.............OK
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state..OK
+ sleep 5
+ '[' -z '' ']'
+ desc 'check backup and restore -- aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.ptuoKcKvQQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.li9lwsfh4J
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ptuoKcKvQQ
++ cat /tmp/tmp.li9lwsfh4J
++ rm /tmp/tmp.ptuoKcKvQQ /tmp/tmp.li9lwsfh4J
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
+ setup_aws_credentials
+ local secret_name=aws-s3-secret
+ [[ -n AKIARXP3OARBMEQ6FSEW ]]
+ [[ -n gljUukL3/TuQnzrtJUgiURH6vqjt+SEo89jGS2Z/ ]]
+ echo 'AWS credentials already set in environment'
AWS credentials already set in environment
+ return 0
+ check_backup_existence_aws operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded/2026-06-11T12:29:44Z/rs0/myApp.test.gz
+ echo 'Backup operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z//rs0/myApp.test.gz found in aws-s3'
Backup operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z//rs0/myApp.test.gz found in aws-s3
+ check_backup_existence_aws operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z /rs1/myApp1.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
+ key=/rs1/myApp1.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded/2026-06-11T12:29:44Z/rs1/myApp1.test.gz
+ echo 'Backup operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z//rs1/myApp1.test.gz found in aws-s3'
Backup operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z//rs1/myApp1.test.gz found in aws-s3
+ check_backup_existence_aws operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z /rs2/myApp2.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
+ key=/rs2/myApp2.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded/2026-06-11T12:29:44Z/rs2/myApp2.test.gz
+ echo 'Backup operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z//rs2/myApp2.test.gz found in aws-s3'
Backup operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z//rs2/myApp2.test.gz found in aws-s3
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7SdnG8j1ak
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Cih7Cs91OK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7SdnG8j1ak
++ cat /tmp/tmp.Cih7Cs91OK
++ rm /tmp/tmp.7SdnG8j1ak /tmp/tmp.Cih7Cs91OK
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3AFv5R7EXA
++ mktemp
+ local LAST_ERR=/tmp/tmp.WuvfJxUyqD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3AFv5R7EXA
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("94027e12-5312-4d45-8d4e-1fc802338a73") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.WuvfJxUyqD
+ rm /tmp/tmp.3AFv5R7EXA /tmp/tmp.WuvfJxUyqD
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9loALXE8CX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B0N0kASTmr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9loALXE8CX
++ cat /tmp/tmp.B0N0kASTmr
++ rm /tmp/tmp.9loALXE8CX /tmp/tmp.B0N0kASTmr
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.gTlmoj79FR
++ mktemp
+ local LAST_ERR=/tmp/tmp.rLWOGkwVSW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gTlmoj79FR
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("1e2e3d86-619d-417b-995c-3fa6ab747087") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.rLWOGkwVSW
+ rm /tmp/tmp.gTlmoj79FR /tmp/tmp.rLWOGkwVSW
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UNknR7rdEP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LYRx68QdkL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UNknR7rdEP
++ cat /tmp/tmp.LYRx68QdkL
++ rm /tmp/tmp.UNknR7rdEP /tmp/tmp.LYRx68QdkL
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.uipniiU7Bc
++ mktemp
+ local LAST_ERR=/tmp/tmp.DoDZrJfElM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uipniiU7Bc
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("1c210286-e766-4bbe-a329-2b7913a8d069") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.DoDZrJfElM
+ rm /tmp/tmp.uipniiU7Bc /tmp/tmp.DoDZrJfElM
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-11T12:31:35+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7a4RP0GWkO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lNMRp3Nk9d
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7a4RP0GWkO
++ cat /tmp/tmp.lNMRp3Nk9d
++ rm /tmp/tmp.7a4RP0GWkO /tmp/tmp.lNMRp3Nk9d
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jOwWFoHvFd
++ mktemp
+ local LAST_ERR=/tmp/tmp.wdD7uTbGsY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jOwWFoHvFd
+ cat /tmp/tmp.wdD7uTbGsY
+ rm /tmp/tmp.jOwWFoHvFd /tmp/tmp.wdD7uTbGsY
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.KZJaOOCJa7/find-2nd
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-11T12:31:37+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D2UFjQFDnu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PuPX081xt3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D2UFjQFDnu
++ cat /tmp/tmp.PuPX081xt3
++ rm /tmp/tmp.D2UFjQFDnu /tmp/tmp.PuPX081xt3
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.scYQhbKXsT
++ mktemp
+ local LAST_ERR=/tmp/tmp.5ZVOg6RASI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.scYQhbKXsT
+ cat /tmp/tmp.5ZVOg6RASI
+ rm /tmp/tmp.scYQhbKXsT /tmp/tmp.5ZVOg6RASI
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.KZJaOOCJa7/find-2nd1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-11T12:31:40+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hDynN5sv0P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uu9HxTJ2mW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hDynN5sv0P
++ cat /tmp/tmp.uu9HxTJ2mW
++ rm /tmp/tmp.hDynN5sv0P /tmp/tmp.uu9HxTJ2mW
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.76d5uDOWMi
++ mktemp
+ local LAST_ERR=/tmp/tmp.eQJR3DGwXk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.76d5uDOWMi
+ cat /tmp/tmp.eQJR3DGwXk
+ rm /tmp/tmp.76d5uDOWMi /tmp/tmp.eQJR3DGwXk
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.KZJaOOCJa7/find-2nd2
+ run_restore backup-aws-s3
+ local backup_name=backup-aws-s3
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/restore.yml
+ log 'running restore restore-backup-aws-s3'
+ set +o xtrace
[2026-06-11T12:31:43+0000] running restore restore-backup-aws-s3
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fOORgusvca
++ mktemp
+ local LAST_ERR=/tmp/tmp.sHybgeoUfW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fOORgusvca
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
+ cat /tmp/tmp.sHybgeoUfW
+ rm /tmp/tmp.fOORgusvca /tmp/tmp.sHybgeoUfW
+ return 0
+ wait_restore backup-aws-s3 some-name
+ local backup_name=backup-aws-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.seRki3PktF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s2mB86ditr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.seRki3PktF
++ cat /tmp/tmp.s2mB86ditr
++ rm /tmp/tmp.seRki3PktF /tmp/tmp.s2mB86ditr
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GiGatrWCY7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gw81CZND6x
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GiGatrWCY7
++ cat /tmp/tmp.gw81CZND6x
++ rm /tmp/tmp.GiGatrWCY7 /tmp/tmp.gw81CZND6x
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6HYRvcSe8z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8VUHHzsQcP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6HYRvcSe8z
++ cat /tmp/tmp.8VUHHzsQcP
++ rm /tmp/tmp.6HYRvcSe8z /tmp/tmp.8VUHHzsQcP
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DmSMw9YhjB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZHB3J1vbrh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DmSMw9YhjB
++ cat /tmp/tmp.ZHB3J1vbrh
++ rm /tmp/tmp.DmSMw9YhjB /tmp/tmp.ZHB3J1vbrh
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6sTd3FXoDV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XHoRkjejli
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6sTd3FXoDV
++ cat /tmp/tmp.XHoRkjejli
++ rm /tmp/tmp.6sTd3FXoDV /tmp/tmp.XHoRkjejli
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ check_data
+ local postfix=
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-11T12:34:02+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DMPWsGfa5i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dVyEGWUZwP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DMPWsGfa5i
++ cat /tmp/tmp.dVyEGWUZwP
++ rm /tmp/tmp.DMPWsGfa5i /tmp/tmp.dVyEGWUZwP
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.johmC4WueZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.hWlauPKFfe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.johmC4WueZ
+ cat /tmp/tmp.hWlauPKFfe
+ rm /tmp/tmp.johmC4WueZ /tmp/tmp.hWlauPKFfe
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.KZJaOOCJa7/find
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-11T12:34:05+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp1\n db.test.find()'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1KCmlM78Cn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Zh5PdXA0yt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1KCmlM78Cn
++ cat /tmp/tmp.Zh5PdXA0yt
++ rm /tmp/tmp.1KCmlM78Cn /tmp/tmp.Zh5PdXA0yt
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.TQUAxjRMCp
++ mktemp
+ local LAST_ERR=/tmp/tmp.kyTR3qT1V5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TQUAxjRMCp
+ cat /tmp/tmp.kyTR3qT1V5
+ rm /tmp/tmp.TQUAxjRMCp /tmp/tmp.kyTR3qT1V5
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.KZJaOOCJa7/find1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-11T12:34:07+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PubDndWEqT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cobocwARWy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PubDndWEqT
++ cat /tmp/tmp.cobocwARWy
++ rm /tmp/tmp.PubDndWEqT /tmp/tmp.cobocwARWy
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.zdwibZ2HCl
++ mktemp
+ local LAST_ERR=/tmp/tmp.v39YufDSaM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zdwibZ2HCl
+ cat /tmp/tmp.v39YufDSaM
+ rm /tmp/tmp.zdwibZ2HCl /tmp/tmp.v39YufDSaM
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.KZJaOOCJa7/find2
+ desc 'check backup and restore -- gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs
++ local backup_name=backup-gcp-cs
++ kubectl_bin get psmdb-backup backup-gcp-cs -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.jkOt83s9lI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OQzXvWiFzR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jkOt83s9lI
++ cat /tmp/tmp.OQzXvWiFzR
++ rm /tmp/tmp.jkOt83s9lI /tmp/tmp.OQzXvWiFzR
++ return 0
+ backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z
+ setup_gcs_credentials
+ local secret_name=gcp-cs-secret
+ gsutil ls
+ echo 'Setting up GCS credentials from K8s secret: gcp-cs-secret'
Setting up GCS credentials from K8s secret: gcp-cs-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ echo 'GCS credentials configured successfully'
GCS credentials configured successfully
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs0/myApp.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs0/myApp.test.gz found in gcp-cs
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z /rs1/myApp1.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z
+ obj=/rs1/myApp1.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs1/myApp1.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs1/myApp1.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs1/myApp1.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs1/myApp1.test.gz found in gcp-cs
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z /rs2/myApp2.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z
+ obj=/rs2/myApp2.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs2/myApp2.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs2/myApp2.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs2/myApp2.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs2/myApp2.test.gz found in gcp-cs
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BI76PezWeN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VWCK0RI0cm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BI76PezWeN
++ cat /tmp/tmp.VWCK0RI0cm
++ rm /tmp/tmp.BI76PezWeN /tmp/tmp.VWCK0RI0cm
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.KfjZ7aV3dZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.JcMMLlGBE4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KfjZ7aV3dZ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("54f1d00b-b10d-4a4b-9e80-535b8f3db6bb") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.JcMMLlGBE4
+ rm /tmp/tmp.KfjZ7aV3dZ /tmp/tmp.JcMMLlGBE4
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VnAmHAh9uw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5eIqhzIGnW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VnAmHAh9uw
++ cat /tmp/tmp.5eIqhzIGnW
++ rm /tmp/tmp.VnAmHAh9uw /tmp/tmp.5eIqhzIGnW
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3nQm0hTDi8
++ mktemp
+ local LAST_ERR=/tmp/tmp.lEXkurwfvY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3nQm0hTDi8
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("54d326b2-6df5-4a08-9203-66ba1491614a") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.lEXkurwfvY
+ rm /tmp/tmp.3nQm0hTDi8 /tmp/tmp.lEXkurwfvY
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UX5mjEQbcj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kdf8XIlTEF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UX5mjEQbcj
++ cat /tmp/tmp.kdf8XIlTEF
++ rm /tmp/tmp.UX5mjEQbcj /tmp/tmp.kdf8XIlTEF
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.iiA2QExGLh
++ mktemp
+ local LAST_ERR=/tmp/tmp.dhLfD3lyfJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iiA2QExGLh
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("94a310e8-b17a-4a89-8c00-16bdef17cc65") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.dhLfD3lyfJ
+ rm /tmp/tmp.iiA2QExGLh /tmp/tmp.dhLfD3lyfJ
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-11T12:34:25+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YZGdDmRSng
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GDQ6wPlwPb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YZGdDmRSng
++ cat /tmp/tmp.GDQ6wPlwPb
++ rm /tmp/tmp.YZGdDmRSng /tmp/tmp.GDQ6wPlwPb
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.J2jE9mkRNu
++ mktemp
+ local LAST_ERR=/tmp/tmp.LXY1pqx2YU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J2jE9mkRNu
+ cat /tmp/tmp.LXY1pqx2YU
+ rm /tmp/tmp.J2jE9mkRNu /tmp/tmp.LXY1pqx2YU
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.KZJaOOCJa7/find-2nd
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-11T12:34:28+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SpDW1BC35i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DwJbDeyWc9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SpDW1BC35i
++ cat /tmp/tmp.DwJbDeyWc9
++ rm /tmp/tmp.SpDW1BC35i /tmp/tmp.DwJbDeyWc9
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.W01K04O6ky
++ mktemp
+ local LAST_ERR=/tmp/tmp.hxUWWAspFh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.W01K04O6ky
+ cat /tmp/tmp.hxUWWAspFh
+ rm /tmp/tmp.W01K04O6ky /tmp/tmp.hxUWWAspFh
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.KZJaOOCJa7/find-2nd1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-11T12:34:30+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mNUiuqAPVA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hWj1aJvSIz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mNUiuqAPVA
++ cat /tmp/tmp.hWj1aJvSIz
++ rm /tmp/tmp.mNUiuqAPVA /tmp/tmp.hWj1aJvSIz
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kdcG1WW8S8
++ mktemp
+ local LAST_ERR=/tmp/tmp.mUydu3pk8i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kdcG1WW8S8
+ cat /tmp/tmp.mUydu3pk8i
+ rm /tmp/tmp.kdcG1WW8S8 /tmp/tmp.mUydu3pk8i
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.KZJaOOCJa7/find-2nd2
+ run_restore backup-gcp-cs
+ local backup_name=backup-gcp-cs
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/restore.yml
+ log 'running restore restore-backup-gcp-cs'
+ set +o xtrace
[2026-06-11T12:34:33+0000] running restore restore-backup-gcp-cs
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs/'
+ kubectl_bin apply -f -
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.gLYzS27dRb
++ mktemp
+ local LAST_ERR=/tmp/tmp.Gjllg1AVXR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gLYzS27dRb
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs created
+ cat /tmp/tmp.Gjllg1AVXR
+ rm /tmp/tmp.gLYzS27dRb /tmp/tmp.Gjllg1AVXR
+ return 0
+ wait_restore backup-gcp-cs some-name
+ local backup_name=backup-gcp-cs
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SfLHg9LcK9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DjemRg8FYY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SfLHg9LcK9
++ cat /tmp/tmp.DjemRg8FYY
++ rm /tmp/tmp.SfLHg9LcK9 /tmp/tmp.DjemRg8FYY
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VNmRaIf24r
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1lJbipTwiT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VNmRaIf24r
++ cat /tmp/tmp.1lJbipTwiT
++ rm /tmp/tmp.VNmRaIf24r /tmp/tmp.1lJbipTwiT
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hUdGvXV0eQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DHhmJOCaPK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hUdGvXV0eQ
++ cat /tmp/tmp.DHhmJOCaPK
++ rm /tmp/tmp.hUdGvXV0eQ /tmp/tmp.DHhmJOCaPK
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Vc1kcNvKqa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q6rVaorDjf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Vc1kcNvKqa
++ cat /tmp/tmp.q6rVaorDjf
++ rm /tmp/tmp.Vc1kcNvKqa /tmp/tmp.q6rVaorDjf
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UzrUpl6JrO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0vBYUZ0EDh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UzrUpl6JrO
++ cat /tmp/tmp.0vBYUZ0EDh
++ rm /tmp/tmp.UzrUpl6JrO /tmp/tmp.0vBYUZ0EDh
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lKEKM6mmVa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PSzbEzDusq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lKEKM6mmVa
++ cat /tmp/tmp.PSzbEzDusq
++ rm /tmp/tmp.lKEKM6mmVa /tmp/tmp.PSzbEzDusq
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ check_data
+ local postfix=
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-11T12:37:04+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mqqKAcHrNh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l9aP91CKNh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mqqKAcHrNh
++ cat /tmp/tmp.l9aP91CKNh
++ rm /tmp/tmp.mqqKAcHrNh /tmp/tmp.l9aP91CKNh
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.tdwNo53kBy
++ mktemp
+ local LAST_ERR=/tmp/tmp.hp0fz3oaBp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tdwNo53kBy
+ cat /tmp/tmp.hp0fz3oaBp
+ rm /tmp/tmp.tdwNo53kBy /tmp/tmp.hp0fz3oaBp
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.KZJaOOCJa7/find
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-11T12:37:06+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wgUerTCmEc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wi23FCZ21O
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wgUerTCmEc
++ cat /tmp/tmp.wi23FCZ21O
++ rm /tmp/tmp.wgUerTCmEc /tmp/tmp.wi23FCZ21O
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VvpefgoErM
++ mktemp
+ local LAST_ERR=/tmp/tmp.22NSKM3Jb6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VvpefgoErM
+ cat /tmp/tmp.22NSKM3Jb6
+ rm /tmp/tmp.VvpefgoErM /tmp/tmp.22NSKM3Jb6
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.KZJaOOCJa7/find1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-11T12:37:09+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jVQodSnmIX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.p1Lx687dDN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jVQodSnmIX
++ cat /tmp/tmp.p1Lx687dDN
++ rm /tmp/tmp.jVQodSnmIX /tmp/tmp.p1Lx687dDN
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.D3ZWvfqLDS
++ mktemp
+ local LAST_ERR=/tmp/tmp.riPETqDPPp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.D3ZWvfqLDS
+ cat /tmp/tmp.riPETqDPPp
+ rm /tmp/tmp.D3ZWvfqLDS /tmp/tmp.riPETqDPPp
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.KZJaOOCJa7/find2
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.h3u8pEsDXV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2dzFsuqpPw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.h3u8pEsDXV
++ cat /tmp/tmp.2dzFsuqpPw
++ rm /tmp/tmp.h3u8pEsDXV /tmp/tmp.2dzFsuqpPw
++ return 0
+ backup_dest_azure=operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
+ setup_azure_credentials
+ local secret_name=azure-secret
+ echo 'Setting up Azure credentials from K8s secret: azure-secret'
Setting up Azure credentials from K8s secret: azure-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ echo 'Azure credentials configured successfully'
Azure credentials configured successfully
+ check_backup_existence_azure operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs0/myApp.test.gz found in azure-blob'
Backup psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs0/myApp.test.gz found in azure-blob
+ check_backup_existence_azure operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z /rs1/myApp1.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
+ blob=/rs1/myApp1.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs1/myApp1.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs1/myApp1.test.gz
+ echo 'Backup psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs1/myApp1.test.gz found in azure-blob'
Backup psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs1/myApp1.test.gz found in azure-blob
+ check_backup_existence_azure operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z /rs2/myApp2.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
+ blob=/rs2/myApp2.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs2/myApp2.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs2/myApp2.test.gz
+ echo 'Backup psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs2/myApp2.test.gz found in azure-blob'
Backup psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs2/myApp2.test.gz found in azure-blob
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CQbs7N8V6C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ryNCfU98Cs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CQbs7N8V6C
++ cat /tmp/tmp.ryNCfU98Cs
++ rm /tmp/tmp.CQbs7N8V6C /tmp/tmp.ryNCfU98Cs
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xoTKfj60E4
++ mktemp
+ local LAST_ERR=/tmp/tmp.woWuqif6dy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xoTKfj60E4
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("2b951034-e12a-4fa7-bacb-4ba2a23e1f4a") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.woWuqif6dy
+ rm /tmp/tmp.xoTKfj60E4 /tmp/tmp.woWuqif6dy
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0uZhSKSHfc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pm3zGgrGzk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0uZhSKSHfc
++ cat /tmp/tmp.pm3zGgrGzk
++ rm /tmp/tmp.0uZhSKSHfc /tmp/tmp.pm3zGgrGzk
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5oPLisqpz5
++ mktemp
+ local LAST_ERR=/tmp/tmp.cy7NdrSaX7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5oPLisqpz5
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("cebc6ced-15ea-4949-a5aa-b34ed8f9ea85") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.cy7NdrSaX7
+ rm /tmp/tmp.5oPLisqpz5 /tmp/tmp.cy7NdrSaX7
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4weLMnOeHC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MswejZvubo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4weLMnOeHC
++ cat /tmp/tmp.MswejZvubo
++ rm /tmp/tmp.4weLMnOeHC /tmp/tmp.MswejZvubo
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Arrh9n6DMC
++ mktemp
+ local LAST_ERR=/tmp/tmp.iJGd2Unhwl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Arrh9n6DMC
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("c0b4d46c-2cad-4e6c-a5fa-797a5b981f55") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.iJGd2Unhwl
+ rm /tmp/tmp.Arrh9n6DMC /tmp/tmp.iJGd2Unhwl
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-11T12:37:24+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F1NtjaNhtS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RpcqpxGScx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.F1NtjaNhtS
++ cat /tmp/tmp.RpcqpxGScx
++ rm /tmp/tmp.F1NtjaNhtS /tmp/tmp.RpcqpxGScx
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4FmYYCoHy2
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dhnxjfr8ZV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4FmYYCoHy2
+ cat /tmp/tmp.Dhnxjfr8ZV
+ rm /tmp/tmp.4FmYYCoHy2 /tmp/tmp.Dhnxjfr8ZV
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.KZJaOOCJa7/find-2nd
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-11T12:37:26+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GIMsRy29xG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.skABWVRzxF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GIMsRy29xG
++ cat /tmp/tmp.skABWVRzxF
++ rm /tmp/tmp.GIMsRy29xG /tmp/tmp.skABWVRzxF
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.PizVVNX5fl
++ mktemp
+ local LAST_ERR=/tmp/tmp.HZkaJo95pI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PizVVNX5fl
+ cat /tmp/tmp.HZkaJo95pI
+ rm /tmp/tmp.PizVVNX5fl /tmp/tmp.HZkaJo95pI
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.KZJaOOCJa7/find-2nd1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-11T12:37:29+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DdWveo7dWe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yfN6V2bfQH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DdWveo7dWe
++ cat /tmp/tmp.yfN6V2bfQH
++ rm /tmp/tmp.DdWveo7dWe /tmp/tmp.yfN6V2bfQH
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.LdqZmYkB1h
++ mktemp
+ local LAST_ERR=/tmp/tmp.esCqMM9obe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LdqZmYkB1h
+ cat /tmp/tmp.esCqMM9obe
+ rm /tmp/tmp.LdqZmYkB1h /tmp/tmp.esCqMM9obe
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.KZJaOOCJa7/find-2nd2
+ run_restore backup-azure-blob
+ local backup_name=backup-azure-blob
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/restore.yml
+ log 'running restore restore-backup-azure-blob'
+ set +o xtrace
[2026-06-11T12:37:32+0000] running restore restore-backup-azure-blob
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.FHwaqvgKMr
++ mktemp
+ local LAST_ERR=/tmp/tmp.mCGb5Adp3p
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FHwaqvgKMr
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
+ cat /tmp/tmp.mCGb5Adp3p
+ rm /tmp/tmp.FHwaqvgKMr /tmp/tmp.mCGb5Adp3p
+ return 0
+ wait_restore backup-azure-blob some-name
+ local backup_name=backup-azure-blob
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob object to be created.OK
Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BnTXi7ylA4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.df7cgFI7o0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BnTXi7ylA4
++ cat /tmp/tmp.df7cgFI7o0
++ rm /tmp/tmp.BnTXi7ylA4 /tmp/tmp.df7cgFI7o0
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IPlxaoOkOI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UXgYpau4TN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IPlxaoOkOI
++ cat /tmp/tmp.UXgYpau4TN
++ rm /tmp/tmp.IPlxaoOkOI /tmp/tmp.UXgYpau4TN
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xfcWawbJ7u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RcDaA7FFEC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xfcWawbJ7u
++ cat /tmp/tmp.RcDaA7FFEC
++ rm /tmp/tmp.xfcWawbJ7u /tmp/tmp.RcDaA7FFEC
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fXIItml2WI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nKw00IGXFG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fXIItml2WI
++ cat /tmp/tmp.nKw00IGXFG
++ rm /tmp/tmp.fXIItml2WI /tmp/tmp.nKw00IGXFG
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7w0Q4s6SNB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J2Dk5Qp0so
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7w0Q4s6SNB
++ cat /tmp/tmp.J2Dk5Qp0so
++ rm /tmp/tmp.7w0Q4s6SNB /tmp/tmp.J2Dk5Qp0so
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ check_data
+ local postfix=
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-11T12:39:51+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6XtmHNWvjf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.259zxrZsNi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6XtmHNWvjf
++ cat /tmp/tmp.259zxrZsNi
++ rm /tmp/tmp.6XtmHNWvjf /tmp/tmp.259zxrZsNi
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ORfWeuFQtO
++ mktemp
+ local LAST_ERR=/tmp/tmp.7rAwvebnuf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ORfWeuFQtO
+ cat /tmp/tmp.7rAwvebnuf
+ rm /tmp/tmp.ORfWeuFQtO /tmp/tmp.7rAwvebnuf
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.KZJaOOCJa7/find
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-11T12:39:54+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.loIVW64nvS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XYt5g05CbK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.loIVW64nvS
++ cat /tmp/tmp.XYt5g05CbK
++ rm /tmp/tmp.loIVW64nvS /tmp/tmp.XYt5g05CbK
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.U5mnYl90su
++ mktemp
+ local LAST_ERR=/tmp/tmp.iWy1tWdEpy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.U5mnYl90su
+ cat /tmp/tmp.iWy1tWdEpy
+ rm /tmp/tmp.U5mnYl90su /tmp/tmp.iWy1tWdEpy
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.KZJaOOCJa7/find1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-11T12:39:56+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.23bDoHVZdR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LXRBd5xywV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.23bDoHVZdR
++ cat /tmp/tmp.LXRBd5xywV
++ rm /tmp/tmp.23bDoHVZdR /tmp/tmp.LXRBd5xywV
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.LxO7fLnC1B
++ mktemp
+ local LAST_ERR=/tmp/tmp.KAqaY4b4bQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LxO7fLnC1B
+ cat /tmp/tmp.KAqaY4b4bQ
+ rm /tmp/tmp.LxO7fLnC1B /tmp/tmp.KAqaY4b4bQ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.KZJaOOCJa7/find2
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

+ check_backup_in_storage backup-minio minio rs0 myApp.test.gz
+ local backup=backup-minio
+ local storage_type=minio
+ local replset=rs0
+ local file=myApp.test.gz
+ local protocol=http
+ local endpoint
++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.ePmoSf6FUT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G87C1OkVaS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ePmoSf6FUT
++ cat /tmp/tmp.G87C1OkVaS
++ rm /tmp/tmp.ePmoSf6FUT /tmp/tmp.G87C1OkVaS
++ return 0
+ backup_dest=operator-testing/2026-06-11T12:29:10Z
+ case ${storage_type} in
+ endpoint=minio-service
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-06-11T12:29:10Z/rs0/myApp.test.gz
+ grep myApp.test.gz
++ mktemp
+ local LAST_OUT=/tmp/tmp.5FndfXsGQL
++ mktemp
+ local LAST_ERR=/tmp/tmp.oUSH0UxHNO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-06-11T12:29:10Z/rs0/myApp.test.gz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5FndfXsGQL
+ cat /tmp/tmp.oUSH0UxHNO
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.5FndfXsGQL /tmp/tmp.oUSH0UxHNO
+ return 0
2026-06-11 12:29:14         56 myApp.test.gz
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WqYLLej4bR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gNgRZKmGRv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WqYLLej4bR
++ cat /tmp/tmp.gNgRZKmGRv
++ rm /tmp/tmp.WqYLLej4bR /tmp/tmp.gNgRZKmGRv
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.IlEmvvtM2y
++ mktemp
+ local LAST_ERR=/tmp/tmp.PKyqycC0HV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IlEmvvtM2y
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("9ba7d239-ee29-4f6e-8732-590d6e3c763e") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.PKyqycC0HV
+ rm /tmp/tmp.IlEmvvtM2y /tmp/tmp.PKyqycC0HV
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Bj1yYc35v5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FtmDAH8jMI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Bj1yYc35v5
++ cat /tmp/tmp.FtmDAH8jMI
++ rm /tmp/tmp.Bj1yYc35v5 /tmp/tmp.FtmDAH8jMI
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.L23PB5vf29
++ mktemp
+ local LAST_ERR=/tmp/tmp.T7XQhtnnCp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.L23PB5vf29
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("0c14b4d9-368d-49aa-8c04-9195c4dfc881") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.T7XQhtnnCp
+ rm /tmp/tmp.L23PB5vf29 /tmp/tmp.T7XQhtnnCp
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.irYrbvv8vU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZWSdjKw0Hm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.irYrbvv8vU
++ cat /tmp/tmp.ZWSdjKw0Hm
++ rm /tmp/tmp.irYrbvv8vU /tmp/tmp.ZWSdjKw0Hm
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fR4wyPuMpM
++ mktemp
+ local LAST_ERR=/tmp/tmp.rumNC8idWi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fR4wyPuMpM
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("0fd51e59-67f4-4a07-82a1-968958315c8f") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.rumNC8idWi
+ rm /tmp/tmp.fR4wyPuMpM /tmp/tmp.rumNC8idWi
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-11T12:40:14+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KDUxmshOdD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KOjbY1F0oq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KDUxmshOdD
++ cat /tmp/tmp.KOjbY1F0oq
++ rm /tmp/tmp.KDUxmshOdD /tmp/tmp.KOjbY1F0oq
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0CF7TVRwhT
++ mktemp
+ local LAST_ERR=/tmp/tmp.kzBWUb16L7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0CF7TVRwhT
+ cat /tmp/tmp.kzBWUb16L7
+ rm /tmp/tmp.0CF7TVRwhT /tmp/tmp.kzBWUb16L7
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.KZJaOOCJa7/find-2nd
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-11T12:40:16+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fLcsEfoeUR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ywiY8Ffe3Z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fLcsEfoeUR
++ cat /tmp/tmp.ywiY8Ffe3Z
++ rm /tmp/tmp.fLcsEfoeUR /tmp/tmp.ywiY8Ffe3Z
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3e0UIjwpWe
++ mktemp
+ local LAST_ERR=/tmp/tmp.sPeQxTEm7r
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3e0UIjwpWe
+ cat /tmp/tmp.sPeQxTEm7r
+ rm /tmp/tmp.3e0UIjwpWe /tmp/tmp.sPeQxTEm7r
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.KZJaOOCJa7/find-2nd1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-11T12:40:19+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WoXSQRACIs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tkI8tY8mdl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WoXSQRACIs
++ cat /tmp/tmp.tkI8tY8mdl
++ rm /tmp/tmp.WoXSQRACIs /tmp/tmp.tkI8tY8mdl
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VVKZ96uf9T
++ mktemp
+ local LAST_ERR=/tmp/tmp.wr5QRxEAwI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VVKZ96uf9T
+ cat /tmp/tmp.wr5QRxEAwI
+ rm /tmp/tmp.VVKZ96uf9T /tmp/tmp.wr5QRxEAwI
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.KZJaOOCJa7/find-2nd2
+ run_restore backup-minio
+ local backup_name=backup-minio
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/restore.yml
+ log 'running restore restore-backup-minio'
+ set +o xtrace
[2026-06-11T12:40:21+0000] running restore restore-backup-minio
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.tdvbJG7DOc
++ mktemp
+ local LAST_ERR=/tmp/tmp.DRnPtrtyHe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tdvbJG7DOc
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
+ cat /tmp/tmp.DRnPtrtyHe
+ rm /tmp/tmp.tdvbJG7DOc /tmp/tmp.DRnPtrtyHe
+ return 0
+ wait_restore backup-minio some-name
+ local backup_name=backup-minio
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio object to be created.OK
Waiting psmdb-restore/restore-backup-minio to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a064EWu9Km
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pk789NA1nb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.a064EWu9Km
++ cat /tmp/tmp.pk789NA1nb
++ rm /tmp/tmp.a064EWu9Km /tmp/tmp.pk789NA1nb
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.etWkhTjRYU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cJiCVjptZh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.etWkhTjRYU
++ cat /tmp/tmp.cJiCVjptZh
++ rm /tmp/tmp.etWkhTjRYU /tmp/tmp.cJiCVjptZh
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ISYui7U97U
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dWHMiBZ0Ky
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ISYui7U97U
++ cat /tmp/tmp.dWHMiBZ0Ky
++ rm /tmp/tmp.ISYui7U97U /tmp/tmp.dWHMiBZ0Ky
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eE7nY1PPnl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mmJLOEkX79
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eE7nY1PPnl
++ cat /tmp/tmp.mmJLOEkX79
++ rm /tmp/tmp.eE7nY1PPnl /tmp/tmp.mmJLOEkX79
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rpaJiTMyCu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cWDgrva7zJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rpaJiTMyCu
++ cat /tmp/tmp.cWDgrva7zJ
++ rm /tmp/tmp.rpaJiTMyCu /tmp/tmp.cWDgrva7zJ
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6PcCKHW6pY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MRfHohDcDD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6PcCKHW6pY
++ cat /tmp/tmp.MRfHohDcDD
++ rm /tmp/tmp.6PcCKHW6pY /tmp/tmp.MRfHohDcDD
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ check_data
+ local postfix=
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-11T12:42:46+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qAlO4QdAn9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6ksI06I69I
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qAlO4QdAn9
++ cat /tmp/tmp.6ksI06I69I
++ rm /tmp/tmp.qAlO4QdAn9 /tmp/tmp.6ksI06I69I
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DX46flOhM7
++ mktemp
+ local LAST_ERR=/tmp/tmp.LKWb9CiLwY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DX46flOhM7
+ cat /tmp/tmp.LKWb9CiLwY
+ rm /tmp/tmp.DX46flOhM7 /tmp/tmp.LKWb9CiLwY
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.KZJaOOCJa7/find
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-11T12:42:48+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Mr241MU1aM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IDMeBHY4rg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Mr241MU1aM
++ cat /tmp/tmp.IDMeBHY4rg
++ rm /tmp/tmp.Mr241MU1aM /tmp/tmp.IDMeBHY4rg
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8T0GnGG0xM
++ mktemp
+ local LAST_ERR=/tmp/tmp.7ycu8AKZET
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8T0GnGG0xM
+ cat /tmp/tmp.7ycu8AKZET
+ rm /tmp/tmp.8T0GnGG0xM /tmp/tmp.7ycu8AKZET
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.KZJaOOCJa7/find1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-429 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-11T12:42:51+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-429 mongodb .svc.cluster.local '' 27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-429
+ local driver=mongodb
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dFG0Dtf92O
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5Uoewb76aO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dFG0Dtf92O
++ cat /tmp/tmp.5Uoewb76aO
++ rm /tmp/tmp.dFG0Dtf92O /tmp/tmp.5Uoewb76aO
++ return 0
+ local client_container=psmdb-client-bb8b97679-5sxqm
+ kubectl_bin exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.t2fiQlYQZI
++ mktemp
+ local LAST_ERR=/tmp/tmp.rUlOOBSFPC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-5sxqm -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-429.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t2fiQlYQZI
+ cat /tmp/tmp.rUlOOBSFPC
+ rm /tmp/tmp.t2fiQlYQZI /tmp/tmp.rUlOOBSFPC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.KZJaOOCJa7/find2
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.GSlhKiGmGl
++ mktemp
+ local LAST_ERR=/tmp/tmp.5rtUodF9XE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GSlhKiGmGl
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-sharded-429 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-sharded-429 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs" deleted from demand-backup-sharded-429 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-sharded-429 namespace
+ cat /tmp/tmp.5rtUodF9XE
+ rm /tmp/tmp.GSlhKiGmGl /tmp/tmp.5rtUodF9XE
+ return 0
++ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ grep -c _
++ cat
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UzvnhpkxpQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.c5yoc8IoAk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UzvnhpkxpQ
++ cat /tmp/tmp.c5yoc8IoAk
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-sharded-429
++ rm /tmp/tmp.UzvnhpkxpQ /tmp/tmp.c5yoc8IoAk
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion_aws operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup-sharded/2026-06-11T12:29:44Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded/2026-06-11T12:29:44Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup-sharded/2026-06-11T12:29:44Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3'
Backup psmdb-demand-backup-sharded/2026-06-11T12:29:44Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:18Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_azure operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup-sharded/2026-06-11T12:30:52Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob'
Backup psmdb-demand-backup-sharded/2026-06-11T12:30:52Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/conf/container-rc.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.aXYEGVDOXR
++ mktemp
+ local LAST_ERR=/tmp/tmp.SGGdlulpIZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/e2e-tests/conf/container-rc.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aXYEGVDOXR
runtimeclass.node.k8s.io "container-rc" deleted
+ cat /tmp/tmp.SGGdlulpIZ
+ rm /tmp/tmp.aXYEGVDOXR /tmp/tmp.SGGdlulpIZ
+ return 0
+ destroy demand-backup-sharded-429
+ local namespace=demand-backup-sharded-429
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9m0KOxiup9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RnjenhsWnz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9m0KOxiup9
++ cat /tmp/tmp.RnjenhsWnz
No resources found in demand-backup-sharded-429 namespace.
++ rm /tmp/tmp.9m0KOxiup9 /tmp/tmp.RnjenhsWnz
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.aaTiR3KEzi
++ mktemp
+ local LAST_ERR=/tmp/tmp.3iHFpQLf6l
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aaTiR3KEzi
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.3iHFpQLf6l
+ rm /tmp/tmp.aaTiR3KEzi /tmp/tmp.3iHFpQLf6l
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.haVZzcLO64
++ mktemp
+ local LAST_ERR=/tmp/tmp.vt4hUbEaE8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.haVZzcLO64
+ cat /tmp/tmp.vt4hUbEaE8
+ rm /tmp/tmp.haVZzcLO64 /tmp/tmp.vt4hUbEaE8
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.uVVb6dFbtg
++ mktemp
+ local LAST_ERR=/tmp/tmp.46cnEkakqo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uVVb6dFbtg
+ cat /tmp/tmp.46cnEkakqo
+ rm /tmp/tmp.uVVb6dFbtg /tmp/tmp.46cnEkakqo
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.qWA7H0yWcl
++ mktemp
+ local LAST_ERR=/tmp/tmp.deekQXx1XP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qWA7H0yWcl
+ cat /tmp/tmp.deekQXx1XP
+ rm /tmp/tmp.qWA7H0yWcl /tmp/tmp.deekQXx1XP
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.AXD90W2g7n
++ mktemp
+ local LAST_ERR=/tmp/tmp.OmvhGDPgNj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2381/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AXD90W2g7n
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.OmvhGDPgNj
+ rm /tmp/tmp.AXD90W2g7n /tmp/tmp.OmvhGDPgNj
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.H9MSpfKVBJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.9fGPgW8mmE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.H9MSpfKVBJ
+ cat /tmp/tmp.9fGPgW8mmE
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.H9MSpfKVBJ
+ cat /tmp/tmp.9fGPgW8mmE
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.H9MSpfKVBJ
+ cat /tmp/tmp.9fGPgW8mmE
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.H9MSpfKVBJ
+ cat /tmp/tmp.9fGPgW8mmE
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.H9MSpfKVBJ /tmp/tmp.9fGPgW8mmE
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ rm -rf /tmp/tmp.KZJaOOCJa7
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-sharded-429
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.E1dsBXOeMu
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.DCHYC4iP72
++ mktemp
+ local LAST_ERR=/tmp/tmp.ue4WQkdV8x
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.vre2Tw7J15
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-sharded-429