++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/logs/demand-backup-sharded.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/logs/demand-backup-sharded.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ kubectl get nodes
++ grep '^minikube'
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep -eks-
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
++ '[' ']'
++ EKS=0
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep gke
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
++ '[' v1.33.11-gke.1197000 ']'
++ GKE=1
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ /usr/sbin/sed -r 's/[^0-9.]+//g'
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.33
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ custom_port=27019
+ create_infra demand-backup-sharded-3020
+ local ns=demand-backup-sharded-3020
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.2baPvGXkcP
++ mktemp
+ local LAST_ERR=/tmp/tmp.tPETSR2PzH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2baPvGXkcP
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.tPETSR2PzH
+ rm /tmp/tmp.2baPvGXkcP /tmp/tmp.tPETSR2PzH
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-sharded-15045 backup-aws-s3 --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "backup-aws-s3" not found
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.SzgvzhbzC2
++ mktemp
+ local LAST_ERR=/tmp/tmp.V6FEF92S7v
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SzgvzhbzC2
+ cat /tmp/tmp.V6FEF92S7v
+ rm /tmp/tmp.SzgvzhbzC2 /tmp/tmp.V6FEF92S7v
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.h7QaH4MNSG
++ mktemp
+ local LAST_ERR=/tmp/tmp.i2rDkp9TgS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.h7QaH4MNSG
+ cat /tmp/tmp.i2rDkp9TgS
+ rm /tmp/tmp.h7QaH4MNSG /tmp/tmp.i2rDkp9TgS
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.5hQd5u922X
++ mktemp
+ local LAST_ERR=/tmp/tmp.ELUbaCxHSM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5hQd5u922X
+ cat /tmp/tmp.ELUbaCxHSM
+ rm /tmp/tmp.5hQd5u922X /tmp/tmp.ELUbaCxHSM
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Jf2OqyXM3i
++ mktemp
+ local LAST_ERR=/tmp/tmp.XjiYn7iCwE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Jf2OqyXM3i
+ cat /tmp/tmp.XjiYn7iCwE
+ rm /tmp/tmp.Jf2OqyXM3i /tmp/tmp.XjiYn7iCwE
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.1CUQdwErm4
++ mktemp
+ local LAST_ERR=/tmp/tmp.womy8kWC1q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1CUQdwErm4
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.womy8kWC1q
+ rm /tmp/tmp.1CUQdwErm4 /tmp/tmp.womy8kWC1q
+ return 0
+ check_crd_for_deletion PR-2406-1c5b15f05
+ local git_tag=PR-2406-1c5b15f05
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2406-1c5b15f05/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.A4IXgHUTwY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wl4ZxhzKDQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.A4IXgHUTwY
++ cat /tmp/tmp.wl4ZxhzKDQ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.A4IXgHUTwY
++ cat /tmp/tmp.wl4ZxhzKDQ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.A4IXgHUTwY
++ cat /tmp/tmp.wl4ZxhzKDQ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.A4IXgHUTwY
++ cat /tmp/tmp.wl4ZxhzKDQ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.A4IXgHUTwY /tmp/tmp.wl4ZxhzKDQ
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace
+ awk '{print$1}'


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.i9S7EVlPKW
+ local LAST_OUT=/tmp/tmp.DWbUOJxgJh
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.Lrpn14OWIN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.mKtu4TiaGs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.i9S7EVlPKW
+ cat /tmp/tmp.Lrpn14OWIN
+ rm /tmp/tmp.i9S7EVlPKW /tmp/tmp.Lrpn14OWIN
+ return 0
namespace "demand-backup-sharded-15045" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DWbUOJxgJh
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.mKtu4TiaGs
+ rm /tmp/tmp.DWbUOJxgJh /tmp/tmp.mKtu4TiaGs
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.a2ttD1RZ9y
++ mktemp
+ local LAST_ERR=/tmp/tmp.VmTNg0yuHe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.a2ttD1RZ9y
+ cat /tmp/tmp.VmTNg0yuHe
+ rm /tmp/tmp.a2ttD1RZ9y /tmp/tmp.VmTNg0yuHe
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.aXcAPO1tqb
++ mktemp
+ local LAST_ERR=/tmp/tmp.T0Zdp6iHpH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aXcAPO1tqb
namespace/psmdb-operator created
+ cat /tmp/tmp.T0Zdp6iHpH
+ rm /tmp/tmp.aXcAPO1tqb /tmp/tmp.T0Zdp6iHpH
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U6Q6oyA6Xf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GqNH7OteZF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.U6Q6oyA6Xf
++ cat /tmp/tmp.GqNH7OteZF
++ rm /tmp/tmp.U6Q6oyA6Xf /tmp/tmp.GqNH7OteZF
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2406-1c5b15f05-8-cluster6 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.QMUdXNCmSi
++ mktemp
+ local LAST_ERR=/tmp/tmp.2IAaqa5kuU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2406-1c5b15f05-8-cluster6 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QMUdXNCmSi
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2406-1c5b15f05-8-cluster6" modified.
+ cat /tmp/tmp.2IAaqa5kuU
+ rm /tmp/tmp.QMUdXNCmSi /tmp/tmp.2IAaqa5kuU
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2406-1c5b15f05'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2406-1c5b15f05
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.D9yUbePtXi
++ mktemp
+ local LAST_ERR=/tmp/tmp.exp9YnZ6vH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.D9yUbePtXi
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.exp9YnZ6vH
+ rm /tmp/tmp.D9yUbePtXi /tmp/tmp.exp9YnZ6vH
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.5eiJCu1pnF
++ mktemp
+ local LAST_ERR=/tmp/tmp.7SdW5HwqdF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5eiJCu1pnF
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.7SdW5HwqdF
+ rm /tmp/tmp.5eiJCu1pnF /tmp/tmp.7SdW5HwqdF
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2406-1c5b15f05") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.naNY7V6gPZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.0l8mMEDLk0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.naNY7V6gPZ
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.0l8mMEDLk0
+ rm /tmp/tmp.naNY7V6gPZ /tmp/tmp.0l8mMEDLk0
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FJuJ49W5rv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OvrhT3Ti3r
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FJuJ49W5rv
++ cat /tmp/tmp.OvrhT3Ti3r
++ rm /tmp/tmp.FJuJ49W5rv /tmp/tmp.OvrhT3Ti3r
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-f74db4f97-l99f5
+ local pod=percona-server-mongodb-operator-f74db4f97-l99f5
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-f74db4f97-l99f5 to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.naRrRG3hgJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.flbjwSmQVf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.naRrRG3hgJ
++ cat /tmp/tmp.flbjwSmQVf
++ rm /tmp/tmp.naRrRG3hgJ /tmp/tmp.flbjwSmQVf
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-f74db4f97-l99f5
++ mktemp
+ local LAST_OUT=/tmp/tmp.3AXQi3Yegq
++ mktemp
+ local LAST_ERR=/tmp/tmp.WZ3j5qnXdK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-f74db4f97-l99f5
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3AXQi3Yegq
+ cat /tmp/tmp.WZ3j5qnXdK
+ rm /tmp/tmp.3AXQi3Yegq /tmp/tmp.WZ3j5qnXdK
+ return 0
2026-06-18T19:35:42.741Z	INFO	setup	Manager starting up	{"gitCommit": "1c5b15f0523892e2a9831596c60cb774f11b7aff", "gitBranch": "PR-2406-1c5b15f05", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-sharded-3020
+ local namespace=demand-backup-sharded-3020
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces demand-backup-sharded-3020'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-sharded-3020
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-sharded-3020 --ignore-not-found
+ local LAST_OUT=/tmp/tmp.0uDTFzOqPp
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.BbXRGXqABQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.VcLTrNd1kk
++ mktemp
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.JMwiWQSgOu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-sharded-3020 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0uDTFzOqPp
+ cat /tmp/tmp.BbXRGXqABQ
+ rm /tmp/tmp.0uDTFzOqPp /tmp/tmp.BbXRGXqABQ
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VcLTrNd1kk
+ cat /tmp/tmp.JMwiWQSgOu
+ rm /tmp/tmp.VcLTrNd1kk /tmp/tmp.JMwiWQSgOu
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-sharded-3020
++ mktemp
+ local LAST_OUT=/tmp/tmp.6dm4SEAMtx
++ mktemp
+ local LAST_ERR=/tmp/tmp.1rujoeOKYD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-sharded-3020
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6dm4SEAMtx
+ cat /tmp/tmp.1rujoeOKYD
+ rm /tmp/tmp.6dm4SEAMtx /tmp/tmp.1rujoeOKYD
+ return 0
+ desc 'create namespace demand-backup-sharded-3020'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-sharded-3020
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-sharded-3020
++ mktemp
+ local LAST_OUT=/tmp/tmp.SQtd1FG5Ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.5SJAA2Xxuh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-sharded-3020
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SQtd1FG5Ns
namespace/demand-backup-sharded-3020 created
+ cat /tmp/tmp.5SJAA2Xxuh
+ rm /tmp/tmp.SQtd1FG5Ns /tmp/tmp.5SJAA2Xxuh
+ return 0
+ set_kube_ctx demand-backup-sharded-3020
+ local namespace=demand-backup-sharded-3020
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5NX9PRdink
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sIIhI2s2If
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5NX9PRdink
++ cat /tmp/tmp.sIIhI2s2If
++ rm /tmp/tmp.5NX9PRdink /tmp/tmp.sIIhI2s2If
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2406-1c5b15f05-8-cluster6 --namespace=demand-backup-sharded-3020
++ mktemp
+ local LAST_OUT=/tmp/tmp.wqTjguKBtR
++ mktemp
+ local LAST_ERR=/tmp/tmp.1y5MRkgeEA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2406-1c5b15f05-8-cluster6 --namespace=demand-backup-sharded-3020
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wqTjguKBtR
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2406-1c5b15f05-8-cluster6" modified.
+ cat /tmp/tmp.1y5MRkgeEA
+ rm /tmp/tmp.wqTjguKBtR /tmp/tmp.1y5MRkgeEA
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Thu Jun 18 19:36:21 2026
NAMESPACE: demand-backup-sharded-3020
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-sharded-3020.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-3020 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-3020

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-sharded-3020 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-sharded-3020 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0PQteTbJEm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v9G22ddwcB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0PQteTbJEm
++ cat /tmp/tmp.v9G22ddwcB
++ rm /tmp/tmp.0PQteTbJEm /tmp/tmp.v9G22ddwcB
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-bvk5d
+ wait_pod minio-service-6d5f646cdc-bvk5d
+ local pod=minio-service-6d5f646cdc-bvk5d
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-bvk5d to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-sharded-3020.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.MYPY10swu8
++ mktemp
+ local LAST_ERR=/tmp/tmp.jFaHCzuP4v
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MYPY10swu8
make_bucket: operator-testing
make_bucket: operator-testing
pod "aws-cli" deleted from demand-backup-sharded-3020 namespace
+ cat /tmp/tmp.jFaHCzuP4v
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: Internal error occurred: error attaching to container: container is in CONTAINER_EXITED state
+ rm /tmp/tmp.MYPY10swu8 /tmp/tmp.jFaHCzuP4v
+ return 0
+ desc 'create first PSMDB cluster with custom port 27019'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster with custom port 27019
-----------------------------------------------------------------------------------

+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.g0Ut7OLLXm
++ mktemp
+ local LAST_ERR=/tmp/tmp.VWKPdbRMVO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.g0Ut7OLLXm
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.VWKPdbRMVO
+ rm /tmp/tmp.g0Ut7OLLXm /tmp/tmp.VWKPdbRMVO
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZrvIdESgDG
++ mktemp
+ local LAST_ERR=/tmp/tmp.iYbDdwOKPx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZrvIdESgDG
secret/minio-secret created
secret/aws-s3-secret created
secret/alibaba-cloud-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.iYbDdwOKPx
+ rm /tmp/tmp.ZrvIdESgDG /tmp/tmp.iYbDdwOKPx
+ return 0
+ version_gt 1.19
++ echo '1.33 >= 1.19'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' 0 -ne 1 ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/container-rc.yaml
+ /usr/sbin/sed s/docker/runc/g
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.gTRBguCdPV
++ mktemp
+ local LAST_ERR=/tmp/tmp.W30hkintnz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gTRBguCdPV
runtimeclass.node.k8s.io/container-rc unchanged
+ cat /tmp/tmp.W30hkintnz
+ rm /tmp/tmp.gTRBguCdPV /tmp/tmp.W30hkintnz
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml
++ mktemp
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ local LAST_OUT=/tmp/tmp.TeiDAeDEwD
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2406-1c5b15f05"'
+ yq eval '(.spec | select(has("search"))).search.image = "mongodb/mongodb-community-search:latest"'
+ local LAST_ERR=/tmp/tmp.1mAgwD334V
+ local exit_status=0
+ local timeout=4
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ seq 0 2
+ /usr/sbin/sed -e s/NAME_SPACE/demand-backup-sharded-3020/g
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TeiDAeDEwD
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.1mAgwD334V
+ rm /tmp/tmp.TeiDAeDEwD /tmp/tmp.1mAgwD334V
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZQVoUUnnxF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fp7dkLvKmr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZQVoUUnnxF
++ cat /tmp/tmp.fp7dkLvKmr
++ rm /tmp/tmp.ZQVoUUnnxF /tmp/tmp.fp7dkLvKmr
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mE2Yiuv5T4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uNm2jngAl3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mE2Yiuv5T4
++ cat /tmp/tmp.uNm2jngAl3
++ rm /tmp/tmp.mE2Yiuv5T4 /tmp/tmp.uNm2jngAl3
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CfiB4L11su
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6fbtJ8PD6S
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CfiB4L11su
++ cat /tmp/tmp.6fbtJ8PD6S
++ rm /tmp/tmp.CfiB4L11su /tmp/tmp.6fbtJ8PD6S
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness..........................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G9urcHwcPM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SviMrvGRc2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.G9urcHwcPM
++ cat /tmp/tmp.SviMrvGRc2
++ rm /tmp/tmp.G9urcHwcPM /tmp/tmp.SviMrvGRc2
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VrslU91b0V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FG563Ziqy9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VrslU91b0V
++ cat /tmp/tmp.FG563Ziqy9
++ rm /tmp/tmp.VrslU91b0V /tmp/tmp.FG563Ziqy9
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yopBux9KR0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.21Zjf73AY0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yopBux9KR0
++ cat /tmp/tmp.21Zjf73AY0
++ rm /tmp/tmp.yopBux9KR0 /tmp/tmp.21Zjf73AY0
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ false == true ]]
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y1UaPg8fiV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iOrTSc3Gpb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y1UaPg8fiV
++ cat /tmp/tmp.iOrTSc3Gpb
++ rm /tmp/tmp.y1UaPg8fiV /tmp/tmp.iOrTSc3Gpb
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FcFHQquc5U
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ea4HA9H67B
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FcFHQquc5U
++ cat /tmp/tmp.ea4HA9H67B
++ rm /tmp/tmp.FcFHQquc5U /tmp/tmp.ea4HA9H67B
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bQvwskNMrm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Eij0I8jVT2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bQvwskNMrm
++ cat /tmp/tmp.Eij0I8jVT2
++ rm /tmp/tmp.bQvwskNMrm /tmp/tmp.Eij0I8jVT2
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-3020", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.blUWnUhA22
++ mktemp
+ local LAST_ERR=/tmp/tmp.t4lgCuM5XW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.blUWnUhA22
+ cat /tmp/tmp.t4lgCuM5XW
+ rm /tmp/tmp.blUWnUhA22 /tmp/tmp.t4lgCuM5XW
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-06-18T19:40:44+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ compare_kubectl statefulset/some-name-rs1
+ local resource=statefulset/some-name-rs1
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml
+ local new_result=/tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs1.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs1
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-3020", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.MPCyNPIcu3
++ mktemp
+ local LAST_ERR=/tmp/tmp.V9I4suyR4k
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MPCyNPIcu3
+ cat /tmp/tmp.V9I4suyR4k
+ rm /tmp/tmp.MPCyNPIcu3 /tmp/tmp.V9I4suyR4k
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs1.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs1.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs1.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs1.yml
+ log 'compare_kubectl: statefulset/some-name-rs1 OK'
+ set +o xtrace
[2026-06-18T19:40:46+0000] compare_kubectl: statefulset/some-name-rs1 OK
+ compare_kubectl statefulset/some-name-rs2
+ local resource=statefulset/some-name-rs2
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml
+ local new_result=/tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs2.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs2
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-3020", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.siZx8FcIwc
++ mktemp
+ local LAST_ERR=/tmp/tmp.BuchFHnKJB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs2
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.siZx8FcIwc
+ cat /tmp/tmp.BuchFHnKJB
+ rm /tmp/tmp.siZx8FcIwc /tmp/tmp.BuchFHnKJB
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs2.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs2.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs2.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-rs2.yml
+ log 'compare_kubectl: statefulset/some-name-rs2 OK'
+ set +o xtrace
[2026-06-18T19:40:47+0000] compare_kubectl: statefulset/some-name-rs2 OK
+ compare_kubectl statefulset/some-name-cfg
+ local resource=statefulset/some-name-cfg
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml
+ local new_result=/tmp/tmp.QSQ6VoS7pc/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-3020", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.P0Bh1O068h
++ mktemp
+ local LAST_ERR=/tmp/tmp.ptycLavHyt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.P0Bh1O068h
+ cat /tmp/tmp.ptycLavHyt
+ rm /tmp/tmp.P0Bh1O068h /tmp/tmp.ptycLavHyt
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-cfg.yml
+ log 'compare_kubectl: statefulset/some-name-cfg OK'
+ set +o xtrace
[2026-06-18T19:40:47+0000] compare_kubectl: statefulset/some-name-cfg OK
+ compare_kubectl statefulset/some-name-mongos ''
+ local resource=statefulset/some-name-mongos
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml
+ local new_result=/tmp/tmp.QSQ6VoS7pc/statefulset_some-name-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-mongos
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-3020", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.0vLZiPyfO9
++ mktemp
+ local LAST_ERR=/tmp/tmp.BH5fS7Mmvi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0vLZiPyfO9
+ cat /tmp/tmp.BH5fS7Mmvi
+ rm /tmp/tmp.0vLZiPyfO9 /tmp/tmp.BH5fS7Mmvi
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-mongos.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-mongos.yml
+ log 'compare_kubectl: statefulset/some-name-mongos OK'
+ set +o xtrace
[2026-06-18T19:40:49+0000] compare_kubectl: statefulset/some-name-mongos OK
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.KrYEMiOQ3c
++ mktemp
+ local LAST_ERR=/tmp/tmp.tG2tHvVByD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KrYEMiOQ3c
secret/some-name-mongos created
+ cat /tmp/tmp.tG2tHvVByD
+ rm /tmp/tmp.KrYEMiOQ3c /tmp/tmp.tG2tHvVByD
+ return 0
+ sleep 10
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qjnZ6KNa9o
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XE6csirDvC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qjnZ6KNa9o
++ cat /tmp/tmp.XE6csirDvC
++ rm /tmp/tmp.qjnZ6KNa9o /tmp/tmp.XE6csirDvC
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EO2bDcOQnj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VzyzGG57ve
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EO2bDcOQnj
++ cat /tmp/tmp.VzyzGG57ve
++ rm /tmp/tmp.EO2bDcOQnj /tmp/tmp.VzyzGG57ve
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DbtYkTLCPi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mTfW7dYmSK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DbtYkTLCPi
++ cat /tmp/tmp.mTfW7dYmSK
++ rm /tmp/tmp.DbtYkTLCPi /tmp/tmp.mTfW7dYmSK
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness..................................................................................................
+ compare_kubectl statefulset/some-name-mongos -secret
+ local resource=statefulset/some-name-mongos
+ local postfix=-secret
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml
+ local new_result=/tmp/tmp.QSQ6VoS7pc/statefulset_some-name-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-mongos
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-3020", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.0O3QXELm3o
++ mktemp
+ local LAST_ERR=/tmp/tmp.O3m3epMnOy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0O3QXELm3o
+ cat /tmp/tmp.O3m3epMnOy
+ rm /tmp/tmp.0O3QXELm3o /tmp/tmp.O3m3epMnOy
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-mongos.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml /tmp/tmp.QSQ6VoS7pc/statefulset_some-name-mongos.yml
+ log 'compare_kubectl: statefulset/some-name-mongos OK'
+ set +o xtrace
[2026-06-18T19:44:48+0000] compare_kubectl: statefulset/some-name-mongos OK
+ desc 'checking hookscripts'
+ set +o xtrace


-----------------------------------------------------------------------------------
checking hookscripts
-----------------------------------------------------------------------------------

+ kubectl_bin logs some-name-rs0-0 backup-agent
+ grep -q 'Backup Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.A8zhYIO4B2
++ mktemp
+ local LAST_ERR=/tmp/tmp.fUbA5yFARj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-rs0-0 backup-agent
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A8zhYIO4B2
+ cat /tmp/tmp.fUbA5yFARj
+ rm /tmp/tmp.A8zhYIO4B2 /tmp/tmp.fUbA5yFARj
+ return 0
+ kubectl_bin logs some-name-rs0-0
+ grep -q 'RS0 Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Sg7h8lKRiO
++ mktemp
+ local LAST_ERR=/tmp/tmp.xejlcjWc1l
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-rs0-0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Sg7h8lKRiO
cat: write error: Broken pipe
+ cat /tmp/tmp.xejlcjWc1l
Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init)
+ rm /tmp/tmp.Sg7h8lKRiO /tmp/tmp.xejlcjWc1l
+ return 0
+ kubectl_bin logs some-name-rs1-0 backup-agent
+ grep -q 'Backup Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.IkBTY7ejU3
++ mktemp
+ local LAST_ERR=/tmp/tmp.MxbPuDac5N
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-rs1-0 backup-agent
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IkBTY7ejU3
+ cat /tmp/tmp.MxbPuDac5N
+ rm /tmp/tmp.IkBTY7ejU3 /tmp/tmp.MxbPuDac5N
+ return 0
+ kubectl_bin logs some-name-rs1-0
+ grep -q 'RS1 Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.3aGBycHkuQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.PaGc2Pqrxv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-rs1-0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3aGBycHkuQ
cat: write error: Broken pipe
+ cat /tmp/tmp.PaGc2Pqrxv
Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init)
+ rm /tmp/tmp.3aGBycHkuQ /tmp/tmp.PaGc2Pqrxv
+ return 0
+ kubectl_bin logs some-name-rs2-0 backup-agent
+ grep -q 'Backup Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.xYu4UQYmKF
++ mktemp
+ local LAST_ERR=/tmp/tmp.Fj3MfDPhmz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-rs2-0 backup-agent
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xYu4UQYmKF
+ cat /tmp/tmp.Fj3MfDPhmz
+ rm /tmp/tmp.xYu4UQYmKF /tmp/tmp.Fj3MfDPhmz
+ return 0
+ kubectl_bin logs some-name-rs2-0
+ grep -q 'RS2 Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.9AT6eqquDK
++ mktemp
+ local LAST_ERR=/tmp/tmp.sf3yldVali
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-rs2-0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9AT6eqquDK
cat: write error: Broken pipe
+ cat /tmp/tmp.sf3yldVali
Defaulted container "mongod" out of: mongod, rs-sidecar-1, backup-agent, mongo-init (init)
+ rm /tmp/tmp.9AT6eqquDK /tmp/tmp.sf3yldVali
+ return 0
+ kubectl_bin logs some-name-cfg-0 backup-agent
+ grep -q 'Backup Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.DbgSpqiw1T
++ mktemp
+ local LAST_ERR=/tmp/tmp.FcT9JKRBYs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-cfg-0 backup-agent
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DbgSpqiw1T
+ cat /tmp/tmp.FcT9JKRBYs
+ rm /tmp/tmp.DbgSpqiw1T /tmp/tmp.FcT9JKRBYs
+ return 0
+ kubectl_bin logs some-name-cfg-0
+ grep -q 'CFG Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.AqMLO80mZW
++ mktemp
+ local LAST_ERR=/tmp/tmp.QySdnznblR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-cfg-0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AqMLO80mZW
cat: write error: Broken pipe
+ cat /tmp/tmp.QySdnznblR
Defaulted container "mongod" out of: mongod, cfg-sidecar-1, backup-agent, mongo-init (init)
+ rm /tmp/tmp.AqMLO80mZW /tmp/tmp.QySdnznblR
+ return 0
+ kubectl_bin logs some-name-mongos-0
+ grep -q 'Mongos Hookscript started'
++ mktemp
+ local LAST_OUT=/tmp/tmp.5wpVKn27da
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZzIwOb7hYB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs some-name-mongos-0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5wpVKn27da
cat: write error: Broken pipe
+ cat /tmp/tmp.ZzIwOb7hYB
Defaulted container "mongos" out of: mongos, mongos-sidecar-1, mongo-init (init)
+ rm /tmp/tmp.5wpVKn27da /tmp/tmp.ZzIwOb7hYB
+ return 0
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5OaWgItk22
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RD7Z5zE73B
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5OaWgItk22
++ cat /tmp/tmp.RD7Z5zE73B
++ rm /tmp/tmp.5OaWgItk22 /tmp/tmp.RD7Z5zE73B
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WbB0a9XSFI
++ mktemp
+ local LAST_ERR=/tmp/tmp.mlgAkouOvO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WbB0a9XSFI
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b9380b2a-83ec-4ac7-b5f4-e5c06a6a105e") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		},
		{
			"db" : "myApp1",
			"role" : "readWrite"
		},
		{
			"db" : "myApp2",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.mlgAkouOvO
+ rm /tmp/tmp.WbB0a9XSFI /tmp/tmp.mlgAkouOvO
+ return 0
+ run_mongos 'sh.enableSharding("myApp","rs0")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=sh.enableSharding("myApp","rs0")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nBloWsxhkK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FUmEGY8scq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nBloWsxhkK
++ cat /tmp/tmp.FUmEGY8scq
++ rm /tmp/tmp.nBloWsxhkK /tmp/tmp.FUmEGY8scq
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.TMGJFOnfLj
++ mktemp
+ local LAST_ERR=/tmp/tmp.tsmrsSeII1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TMGJFOnfLj
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("0684c738-e9bd-4517-9d33-bcd3722d2d7a") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1781811907, 9),
		"signature" : {
			"hash" : BinData(0,"5tntAHLLFo1IPzS6a36xEjvMNhA="),
			"keyId" : NumberLong("7652822377834741784")
		}
	},
	"operationTime" : Timestamp(1781811907, 6)
}
bye
+ cat /tmp/tmp.tsmrsSeII1
+ rm /tmp/tmp.TMGJFOnfLj /tmp/tmp.tsmrsSeII1
+ return 0
+ run_mongos 'sh.enableSharding("myApp1","rs1")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=sh.enableSharding("myApp1","rs1")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Pe6MpKI5xl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fzTlEZrH6c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Pe6MpKI5xl
++ cat /tmp/tmp.fzTlEZrH6c
++ rm /tmp/tmp.Pe6MpKI5xl /tmp/tmp.fzTlEZrH6c
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ifTk2Sxsy2
++ mktemp
+ local LAST_ERR=/tmp/tmp.iIQ9e6Q3kz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ifTk2Sxsy2
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b3582b85-5526-43da-b807-4c54c41463dd") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1781811910, 11),
		"signature" : {
			"hash" : BinData(0,"+odIuvYa+sdylBFzV2LLXgfGWiQ="),
			"keyId" : NumberLong("7652822377834741784")
		}
	},
	"operationTime" : Timestamp(1781811910, 8)
}
bye
+ cat /tmp/tmp.iIQ9e6Q3kz
+ rm /tmp/tmp.ifTk2Sxsy2 /tmp/tmp.iIQ9e6Q3kz
+ return 0
+ run_mongos 'sh.enableSharding("myApp2","rs2")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=sh.enableSharding("myApp2","rs2")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.h1oTE93JKE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zdzxGu4JK4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.h1oTE93JKE
++ cat /tmp/tmp.zdzxGu4JK4
++ rm /tmp/tmp.h1oTE93JKE /tmp/tmp.zdzxGu4JK4
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZI06Ml4KfE
++ mktemp
+ local LAST_ERR=/tmp/tmp.DIIrEJ2g2U
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZI06Ml4KfE
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("cdf35c95-5454-47c0-8d69-8a5b1129765f") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1781811913, 8),
		"signature" : {
			"hash" : BinData(0,"BDy/FGOZ6Dkz+ucmWnryBt/T1Gw="),
			"keyId" : NumberLong("7652822377834741784")
		}
	},
	"operationTime" : Timestamp(1781811913, 5)
}
bye
+ cat /tmp/tmp.DIIrEJ2g2U
+ rm /tmp/tmp.ZI06Ml4KfE /tmp/tmp.DIIrEJ2g2U
+ return 0
+ insert_data_mongos 100500 myApp '' 27019
+ local data=100500
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.axy26chKXc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vaOmFYZWlB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.axy26chKXc
++ cat /tmp/tmp.vaOmFYZWlB
++ rm /tmp/tmp.axy26chKXc /tmp/tmp.vaOmFYZWlB
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.uGQ6Bh2XCD
++ mktemp
+ local LAST_ERR=/tmp/tmp.Y0tE6758cy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uGQ6Bh2XCD
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("16ddc979-e78e-42dc-828b-aabeaccce8ce") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Y0tE6758cy
+ rm /tmp/tmp.uGQ6Bh2XCD /tmp/tmp.Y0tE6758cy
+ return 0
+ insert_data_mongos 100500 myApp1 '' 27019
+ local data=100500
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sP8PsyTKa6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fa3W9UMmR5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sP8PsyTKa6
++ cat /tmp/tmp.fa3W9UMmR5
++ rm /tmp/tmp.sP8PsyTKa6 /tmp/tmp.fa3W9UMmR5
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fgysS4b95L
++ mktemp
+ local LAST_ERR=/tmp/tmp.C86mFuljVM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fgysS4b95L
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("3ac8bd37-10a2-42ad-9437-5ef3d4698659") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.C86mFuljVM
+ rm /tmp/tmp.fgysS4b95L /tmp/tmp.C86mFuljVM
+ return 0
+ insert_data_mongos 100500 myApp2 '' 27019
+ local data=100500
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZnqqRNWT3Y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Us1q3HQzMp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZnqqRNWT3Y
++ cat /tmp/tmp.Us1q3HQzMp
++ rm /tmp/tmp.ZnqqRNWT3Y /tmp/tmp.Us1q3HQzMp
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.litefjokdf
++ mktemp
+ local LAST_ERR=/tmp/tmp.zDpAWuOagY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.litefjokdf
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("89dddce6-1fc0-423a-84f9-56ce999e30ef") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.zDpAWuOagY
+ rm /tmp/tmp.litefjokdf /tmp/tmp.zDpAWuOagY
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' '' 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T19:45:20+0000] running db.test.command() in myApp
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb '' '' 27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rEOKSTDKR1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.R96Xm851v1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rEOKSTDKR1
++ cat /tmp/tmp.R96Xm851v1
++ rm /tmp/tmp.rEOKSTDKR1 /tmp/tmp.R96Xm851v1
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qkUjW8H48N
++ mktemp
+ local LAST_ERR=/tmp/tmp.p8AiRTHs3k
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qkUjW8H48N
+ cat /tmp/tmp.p8AiRTHs3k
+ rm /tmp/tmp.qkUjW8H48N /tmp/tmp.p8AiRTHs3k
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.QSQ6VoS7pc/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2026-06-18T19:39:53.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2026-06-18T19:39:55.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2026-06-18T19:39:59.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-0
+ local agent_pod=some-name-rs1-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-0...2026-06-18T19:39:55.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-1
+ local agent_pod=some-name-rs1-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-1...2026-06-18T19:39:56.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-2
+ local agent_pod=some-name-rs1-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-2...2026-06-18T19:40:01.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-0
+ local agent_pod=some-name-rs2-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-0...2026-06-18T19:39:51.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-1
+ local agent_pod=some-name-rs2-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-1...2026-06-18T19:39:51.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-2
+ local agent_pod=some-name-rs2-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-2...2026-06-18T19:40:01.000+0000 I listening for the commands

+ backup_name_aws=backup-aws-s3
+ backup_name_minio=backup-minio
+ backup_name_gcp=backup-gcp-cs
+ backup_name_azure=backup-azure-blob
+ desc 'run backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------

+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ log 'running backup backup-minio'
+ set +o xtrace
[2026-06-18T19:45:35+0000] running backup backup-minio
+ yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.0fLdpLekkw
++ mktemp
+ local LAST_ERR=/tmp/tmp.NlTKrXydfu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0fLdpLekkw
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.NlTKrXydfu
+ rm /tmp/tmp.0fLdpLekkw /tmp/tmp.NlTKrXydfu
+ return 0
+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ log 'running backup backup-aws-s3'
+ set +o xtrace
[2026-06-18T19:45:37+0000] running backup backup-aws-s3
+ yq eval $'.metadata.name = "backup-aws-s3"\n\t\t\t| .spec.storageName = "aws-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.wBvqgH87Cr
++ mktemp
+ local LAST_ERR=/tmp/tmp.FXNPwVevQX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wBvqgH87Cr
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.FXNPwVevQX
+ rm /tmp/tmp.wBvqgH87Cr /tmp/tmp.FXNPwVevQX
+ return 0
+ run_backup gcp-cs
+ local storage=gcp-cs
+ local backup_name=backup-gcp-cs
+ local type=logical
+ log 'running backup backup-gcp-cs'
+ set +o xtrace
[2026-06-18T19:45:39+0000] running backup backup-gcp-cs
+ yq eval $'.metadata.name = "backup-gcp-cs"\n\t\t\t| .spec.storageName = "gcp-cs"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/backup-gcp-cs.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.r4DaS1DwoA
++ mktemp
+ local LAST_ERR=/tmp/tmp.lE33ZiDgf2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.r4DaS1DwoA
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created
+ cat /tmp/tmp.lE33ZiDgf2
+ rm /tmp/tmp.r4DaS1DwoA /tmp/tmp.lE33ZiDgf2
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ log 'running backup backup-azure-blob'
+ set +o xtrace
[2026-06-18T19:45:41+0000] running backup backup-azure-blob
+ yq eval $'.metadata.name = "backup-azure-blob"\n\t\t\t| .spec.storageName = "azure-blob"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/backup-azure-blob.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.5GCihhnypY
++ mktemp
+ local LAST_ERR=/tmp/tmp.EYnUTS0rnG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5GCihhnypY
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.EYnUTS0rnG
+ rm /tmp/tmp.5GCihhnypY /tmp/tmp.EYnUTS0rnG
+ return 0
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state.................................................................OK
+ wait_backup backup-gcp-cs
+ local backup_name=backup-gcp-cs
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs to reach ready state..OK
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state..OK
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state..OK
+ sleep 5
+ '[' -z '' ']'
+ desc 'check backup and restore -- aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|s3://||'
++ local LAST_OUT=/tmp/tmp.2kj8zdp87Z
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_ERR=/tmp/tmp.VvnrPFGLLW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2kj8zdp87Z
++ cat /tmp/tmp.VvnrPFGLLW
++ rm /tmp/tmp.2kj8zdp87Z /tmp/tmp.VvnrPFGLLW
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
+ setup_aws_credentials
+ local secret_name=aws-s3-secret
+ [[ -n AKIARXP3OARBMEQ6FSEW ]]
+ [[ -n gljUukL3/TuQnzrtJUgiURH6vqjt+SEo89jGS2Z/ ]]
+ echo 'AWS credentials already set in environment'
AWS credentials already set in environment
+ return 0
+ check_backup_existence_aws operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded/2026-06-18T19:47:19Z/rs0/myApp.test.gz
+ echo 'Backup operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z//rs0/myApp.test.gz found in aws-s3'
Backup operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z//rs0/myApp.test.gz found in aws-s3
+ check_backup_existence_aws operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z /rs1/myApp1.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
+ key=/rs1/myApp1.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded/2026-06-18T19:47:19Z/rs1/myApp1.test.gz
+ echo 'Backup operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z//rs1/myApp1.test.gz found in aws-s3'
Backup operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z//rs1/myApp1.test.gz found in aws-s3
+ check_backup_existence_aws operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z /rs2/myApp2.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
+ key=/rs2/myApp2.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded/2026-06-18T19:47:19Z/rs2/myApp2.test.gz
+ echo 'Backup operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z//rs2/myApp2.test.gz found in aws-s3'
Backup operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z//rs2/myApp2.test.gz found in aws-s3
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MmLuI5QFQm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5JxFQ8XiXy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MmLuI5QFQm
++ cat /tmp/tmp.5JxFQ8XiXy
++ rm /tmp/tmp.MmLuI5QFQm /tmp/tmp.5JxFQ8XiXy
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fDktZpnL4x
++ mktemp
+ local LAST_ERR=/tmp/tmp.WC98237riS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fDktZpnL4x
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("c65e5bf9-2922-4ce3-8aca-6296953941bb") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.WC98237riS
+ rm /tmp/tmp.fDktZpnL4x /tmp/tmp.WC98237riS
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NY3LJ8KgmT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.66dGRbPzG9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NY3LJ8KgmT
++ cat /tmp/tmp.66dGRbPzG9
++ rm /tmp/tmp.NY3LJ8KgmT /tmp/tmp.66dGRbPzG9
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.bTCRJRu0g4
++ mktemp
+ local LAST_ERR=/tmp/tmp.aFZ8geB5U9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bTCRJRu0g4
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("d64cb189-a826-47e3-bfa4-77f88bf51c18") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.aFZ8geB5U9
+ rm /tmp/tmp.bTCRJRu0g4 /tmp/tmp.aFZ8geB5U9
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1cUEyAMkDQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Cf58e1gMqM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1cUEyAMkDQ
++ cat /tmp/tmp.Cf58e1gMqM
++ rm /tmp/tmp.1cUEyAMkDQ /tmp/tmp.Cf58e1gMqM
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.V3ivfWnnOj
++ mktemp
+ local LAST_ERR=/tmp/tmp.EReKgqjtrl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V3ivfWnnOj
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("46c4dead-6d84-459c-b3a7-1a84a49a43a4") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.EReKgqjtrl
+ rm /tmp/tmp.V3ivfWnnOj /tmp/tmp.EReKgqjtrl
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T19:48:17+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.adZMPZHovK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7djOlM9MdA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.adZMPZHovK
++ cat /tmp/tmp.7djOlM9MdA
++ rm /tmp/tmp.adZMPZHovK /tmp/tmp.7djOlM9MdA
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ORBrBt0xQ7
++ mktemp
+ local LAST_ERR=/tmp/tmp.AMS97kvbPX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ORBrBt0xQ7
+ cat /tmp/tmp.AMS97kvbPX
+ rm /tmp/tmp.ORBrBt0xQ7 /tmp/tmp.AMS97kvbPX
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.QSQ6VoS7pc/find-2nd
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-18T19:48:19+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O489SxV7bo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.j5iqBtJC4A
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.O489SxV7bo
++ cat /tmp/tmp.j5iqBtJC4A
++ rm /tmp/tmp.O489SxV7bo /tmp/tmp.j5iqBtJC4A
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.u5MaGHaXtc
++ mktemp
+ local LAST_ERR=/tmp/tmp.i5OqGs5gBL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.u5MaGHaXtc
+ cat /tmp/tmp.i5OqGs5gBL
+ rm /tmp/tmp.u5MaGHaXtc /tmp/tmp.i5OqGs5gBL
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.QSQ6VoS7pc/find-2nd1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-18T19:48:21+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1HB7N9mtOC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GrCet8U8oF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1HB7N9mtOC
++ cat /tmp/tmp.GrCet8U8oF
++ rm /tmp/tmp.1HB7N9mtOC /tmp/tmp.GrCet8U8oF
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.PR8LlVUsHO
++ mktemp
+ local LAST_ERR=/tmp/tmp.EHo9pM8OWk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PR8LlVUsHO
+ cat /tmp/tmp.EHo9pM8OWk
+ rm /tmp/tmp.PR8LlVUsHO /tmp/tmp.EHo9pM8OWk
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.QSQ6VoS7pc/find-2nd2
+ run_restore backup-aws-s3
+ local backup_name=backup-aws-s3
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/restore.yml
+ log 'running restore restore-backup-aws-s3'
+ set +o xtrace
[2026-06-18T19:48:23+0000] running restore restore-backup-aws-s3
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.CK5pMZYgbk
++ mktemp
+ local LAST_ERR=/tmp/tmp.V5eeknSpg9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CK5pMZYgbk
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
+ cat /tmp/tmp.V5eeknSpg9
+ rm /tmp/tmp.CK5pMZYgbk /tmp/tmp.V5eeknSpg9
+ return 0
+ wait_restore backup-aws-s3 some-name
+ local backup_name=backup-aws-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Dqo1HTOv1L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.27cq4jPdd7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Dqo1HTOv1L
++ cat /tmp/tmp.27cq4jPdd7
++ rm /tmp/tmp.Dqo1HTOv1L /tmp/tmp.27cq4jPdd7
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X8kBoWd5oV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NyOgEqSMku
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.X8kBoWd5oV
++ cat /tmp/tmp.NyOgEqSMku
++ rm /tmp/tmp.X8kBoWd5oV /tmp/tmp.NyOgEqSMku
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O8julmb3ry
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DyXYBDFujE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.O8julmb3ry
++ cat /tmp/tmp.DyXYBDFujE
++ rm /tmp/tmp.O8julmb3ry /tmp/tmp.DyXYBDFujE
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WMmV3zlsps
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Dq4otT0bTw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WMmV3zlsps
++ cat /tmp/tmp.Dq4otT0bTw
++ rm /tmp/tmp.WMmV3zlsps /tmp/tmp.Dq4otT0bTw
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ykg27FahGz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OyAnT94j8g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ykg27FahGz
++ cat /tmp/tmp.OyAnT94j8g
++ rm /tmp/tmp.Ykg27FahGz /tmp/tmp.OyAnT94j8g
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ check_data
+ local postfix=
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T19:50:49+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.En3FERop6U
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dIrGVSZHEY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.En3FERop6U
++ cat /tmp/tmp.dIrGVSZHEY
++ rm /tmp/tmp.En3FERop6U /tmp/tmp.dIrGVSZHEY
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Jko1PwN9hM
++ mktemp
+ local LAST_ERR=/tmp/tmp.hnXatcOQWI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Jko1PwN9hM
+ cat /tmp/tmp.hnXatcOQWI
+ rm /tmp/tmp.Jko1PwN9hM /tmp/tmp.hnXatcOQWI
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.QSQ6VoS7pc/find
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-18T19:50:51+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Pv5tAY3y2j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mSss41Rce0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Pv5tAY3y2j
++ cat /tmp/tmp.mSss41Rce0
++ rm /tmp/tmp.Pv5tAY3y2j /tmp/tmp.mSss41Rce0
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.zMXvUEU41y
++ mktemp
+ local LAST_ERR=/tmp/tmp.VfuZpHv1MM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zMXvUEU41y
+ cat /tmp/tmp.VfuZpHv1MM
+ rm /tmp/tmp.zMXvUEU41y /tmp/tmp.VfuZpHv1MM
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.QSQ6VoS7pc/find1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-18T19:50:54+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zF8Ha0cZYw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.30gGaZIYCz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zF8Ha0cZYw
++ cat /tmp/tmp.30gGaZIYCz
++ rm /tmp/tmp.zF8Ha0cZYw /tmp/tmp.30gGaZIYCz
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ce9LEw2YtS
++ mktemp
+ local LAST_ERR=/tmp/tmp.TG0AeOpN17
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ce9LEw2YtS
+ cat /tmp/tmp.TG0AeOpN17
+ rm /tmp/tmp.ce9LEw2YtS /tmp/tmp.TG0AeOpN17
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.QSQ6VoS7pc/find2
+ desc 'check backup and restore -- gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs
++ local backup_name=backup-gcp-cs
++ kubectl_bin get psmdb-backup backup-gcp-cs -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.YQzQXw3b1q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ytcIoaCa77
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YQzQXw3b1q
++ cat /tmp/tmp.ytcIoaCa77
++ rm /tmp/tmp.YQzQXw3b1q /tmp/tmp.ytcIoaCa77
++ return 0
+ backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z
+ setup_gcs_credentials
+ local secret_name=gcp-cs-secret
+ gsutil ls
+ echo 'Setting up GCS credentials from K8s secret: gcp-cs-secret'
Setting up GCS credentials from K8s secret: gcp-cs-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ echo 'GCS credentials configured successfully'
GCS credentials configured successfully
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs0/myApp.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs0/myApp.test.gz found in gcp-cs
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z /rs1/myApp1.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z
+ obj=/rs1/myApp1.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs1/myApp1.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs1/myApp1.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs1/myApp1.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs1/myApp1.test.gz found in gcp-cs
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z /rs2/myApp2.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z
+ obj=/rs2/myApp2.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs2/myApp2.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs2/myApp2.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs2/myApp2.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs2/myApp2.test.gz found in gcp-cs
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Um1ATf0gsn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BeoL9mykN4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Um1ATf0gsn
++ cat /tmp/tmp.BeoL9mykN4
++ rm /tmp/tmp.Um1ATf0gsn /tmp/tmp.BeoL9mykN4
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OT4Ax9wJuc
++ mktemp
+ local LAST_ERR=/tmp/tmp.0GNGf21FfV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OT4Ax9wJuc
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("6b9e3262-290e-4e84-9197-954f8503e1e7") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.0GNGf21FfV
+ rm /tmp/tmp.OT4Ax9wJuc /tmp/tmp.0GNGf21FfV
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4svtV5f6Fy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.357YgH8mMl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4svtV5f6Fy
++ cat /tmp/tmp.357YgH8mMl
++ rm /tmp/tmp.4svtV5f6Fy /tmp/tmp.357YgH8mMl
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.wPrTa3D7CM
++ mktemp
+ local LAST_ERR=/tmp/tmp.6bzhlGd1RM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wPrTa3D7CM
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("57ceddf6-2679-4dc4-8b69-328549d87706") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.6bzhlGd1RM
+ rm /tmp/tmp.wPrTa3D7CM /tmp/tmp.6bzhlGd1RM
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q92Q3WVL1k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RjCXvI46rr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.q92Q3WVL1k
++ cat /tmp/tmp.RjCXvI46rr
++ rm /tmp/tmp.q92Q3WVL1k /tmp/tmp.RjCXvI46rr
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7Hslr71x7S
++ mktemp
+ local LAST_ERR=/tmp/tmp.vxnO9pFNyV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7Hslr71x7S
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("716f8ca5-850c-41ea-9823-ca8f0f349e2c") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.vxnO9pFNyV
+ rm /tmp/tmp.7Hslr71x7S /tmp/tmp.vxnO9pFNyV
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T19:51:12+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jpZOQiFURv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DkhLOZJVe7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jpZOQiFURv
++ cat /tmp/tmp.DkhLOZJVe7
++ rm /tmp/tmp.jpZOQiFURv /tmp/tmp.DkhLOZJVe7
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.H3UzmPUycY
++ mktemp
+ local LAST_ERR=/tmp/tmp.ifQPEtENCB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.H3UzmPUycY
+ cat /tmp/tmp.ifQPEtENCB
+ rm /tmp/tmp.H3UzmPUycY /tmp/tmp.ifQPEtENCB
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.QSQ6VoS7pc/find-2nd
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-18T19:51:15+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wbVdUgab0E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DSKPElUNiE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wbVdUgab0E
++ cat /tmp/tmp.DSKPElUNiE
++ rm /tmp/tmp.wbVdUgab0E /tmp/tmp.DSKPElUNiE
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FNfOH7CQ1J
++ mktemp
+ local LAST_ERR=/tmp/tmp.AnV7YQqc1H
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FNfOH7CQ1J
+ cat /tmp/tmp.AnV7YQqc1H
+ rm /tmp/tmp.FNfOH7CQ1J /tmp/tmp.AnV7YQqc1H
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.QSQ6VoS7pc/find-2nd1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-18T19:51:18+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oS8mLywazN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HQfGzDmdcR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oS8mLywazN
++ cat /tmp/tmp.HQfGzDmdcR
++ rm /tmp/tmp.oS8mLywazN /tmp/tmp.HQfGzDmdcR
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.j0GT9xTiTv
++ mktemp
+ local LAST_ERR=/tmp/tmp.qnk3fq94wD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.j0GT9xTiTv
+ cat /tmp/tmp.qnk3fq94wD
+ rm /tmp/tmp.j0GT9xTiTv /tmp/tmp.qnk3fq94wD
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.QSQ6VoS7pc/find-2nd2
+ run_restore backup-gcp-cs
+ local backup_name=backup-gcp-cs
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/restore.yml
+ log 'running restore restore-backup-gcp-cs'
+ set +o xtrace
[2026-06-18T19:51:20+0000] running restore restore-backup-gcp-cs
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs/'
+ kubectl_bin apply -f -
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.zvzTe1JvNs
++ mktemp
+ local LAST_ERR=/tmp/tmp.svJatMSE49
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zvzTe1JvNs
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs created
+ cat /tmp/tmp.svJatMSE49
+ rm /tmp/tmp.zvzTe1JvNs /tmp/tmp.svJatMSE49
+ return 0
+ wait_restore backup-gcp-cs some-name
+ local backup_name=backup-gcp-cs
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ly5v1xkCYg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.t8uqNbmP1S
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ly5v1xkCYg
++ cat /tmp/tmp.t8uqNbmP1S
++ rm /tmp/tmp.ly5v1xkCYg /tmp/tmp.t8uqNbmP1S
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tp94PFZNWa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1hOcZ4fMWY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tp94PFZNWa
++ cat /tmp/tmp.1hOcZ4fMWY
++ rm /tmp/tmp.tp94PFZNWa /tmp/tmp.1hOcZ4fMWY
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.utXU0HbszF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DmNUnZV4XD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.utXU0HbszF
++ cat /tmp/tmp.DmNUnZV4XD
++ rm /tmp/tmp.utXU0HbszF /tmp/tmp.DmNUnZV4XD
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K5KgBxP94t
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wTo1yq6m6E
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.K5KgBxP94t
++ cat /tmp/tmp.wTo1yq6m6E
++ rm /tmp/tmp.K5KgBxP94t /tmp/tmp.wTo1yq6m6E
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.r7IODZvwJa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.P1W02H2VmA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.r7IODZvwJa
++ cat /tmp/tmp.P1W02H2VmA
++ rm /tmp/tmp.r7IODZvwJa /tmp/tmp.P1W02H2VmA
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ check_data
+ local postfix=
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T19:53:35+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0z8JMaKJ00
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yWOUks5rkj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0z8JMaKJ00
++ cat /tmp/tmp.yWOUks5rkj
++ rm /tmp/tmp.0z8JMaKJ00 /tmp/tmp.yWOUks5rkj
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vqtvZTwKuk
++ mktemp
+ local LAST_ERR=/tmp/tmp.QDhKGPNkaf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vqtvZTwKuk
+ cat /tmp/tmp.QDhKGPNkaf
+ rm /tmp/tmp.vqtvZTwKuk /tmp/tmp.QDhKGPNkaf
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.QSQ6VoS7pc/find
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-18T19:53:37+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NQu249ULzp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AN6gfV0EUl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NQu249ULzp
++ cat /tmp/tmp.AN6gfV0EUl
++ rm /tmp/tmp.NQu249ULzp /tmp/tmp.AN6gfV0EUl
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OFRvcpFTHm
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ey92z1MGjM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OFRvcpFTHm
+ cat /tmp/tmp.Ey92z1MGjM
+ rm /tmp/tmp.OFRvcpFTHm /tmp/tmp.Ey92z1MGjM
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.QSQ6VoS7pc/find1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-18T19:53:40+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8C6e5NKoSS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TUt0JR2hta
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8C6e5NKoSS
++ cat /tmp/tmp.TUt0JR2hta
++ rm /tmp/tmp.8C6e5NKoSS /tmp/tmp.TUt0JR2hta
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3fOVyEXkQf
++ mktemp
+ local LAST_ERR=/tmp/tmp.xZE3UuERAR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3fOVyEXkQf
+ cat /tmp/tmp.xZE3UuERAR
+ rm /tmp/tmp.3fOVyEXkQf /tmp/tmp.xZE3UuERAR
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.QSQ6VoS7pc/find2
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.ogCNNzhR4p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8XNnLnNncB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ogCNNzhR4p
++ cat /tmp/tmp.8XNnLnNncB
++ rm /tmp/tmp.ogCNNzhR4p /tmp/tmp.8XNnLnNncB
++ return 0
+ backup_dest_azure=operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
+ setup_azure_credentials
+ local secret_name=azure-secret
+ echo 'Setting up Azure credentials from K8s secret: azure-secret'
Setting up Azure credentials from K8s secret: azure-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ echo 'Azure credentials configured successfully'
Azure credentials configured successfully
+ check_backup_existence_azure operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs0/myApp.test.gz found in azure-blob'
Backup psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs0/myApp.test.gz found in azure-blob
+ check_backup_existence_azure operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z /rs1/myApp1.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
+ blob=/rs1/myApp1.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs1/myApp1.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs1/myApp1.test.gz
+ echo 'Backup psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs1/myApp1.test.gz found in azure-blob'
Backup psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs1/myApp1.test.gz found in azure-blob
+ check_backup_existence_azure operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z /rs2/myApp2.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
+ blob=/rs2/myApp2.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs2/myApp2.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs2/myApp2.test.gz
+ echo 'Backup psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs2/myApp2.test.gz found in azure-blob'
Backup psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs2/myApp2.test.gz found in azure-blob
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yHmwzKftQp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6y6iMUzmIF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yHmwzKftQp
++ cat /tmp/tmp.6y6iMUzmIF
++ rm /tmp/tmp.yHmwzKftQp /tmp/tmp.6y6iMUzmIF
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8yglUUHuyv
++ mktemp
+ local LAST_ERR=/tmp/tmp.b2HkEdoGpR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8yglUUHuyv
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e3f83636-7666-402d-bb04-74d4629256ec") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.b2HkEdoGpR
+ rm /tmp/tmp.8yglUUHuyv /tmp/tmp.b2HkEdoGpR
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5ArKacZkEU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2vsm6A8EYU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5ArKacZkEU
++ cat /tmp/tmp.2vsm6A8EYU
++ rm /tmp/tmp.5ArKacZkEU /tmp/tmp.2vsm6A8EYU
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8jm3g6QLk7
++ mktemp
+ local LAST_ERR=/tmp/tmp.5b5XnvKYEJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8jm3g6QLk7
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("6282d270-36d2-4583-a90d-59ccdeb042e3") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.5b5XnvKYEJ
+ rm /tmp/tmp.8jm3g6QLk7 /tmp/tmp.5b5XnvKYEJ
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KfwiunHoeb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jzTy69OlGX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KfwiunHoeb
++ cat /tmp/tmp.jzTy69OlGX
++ rm /tmp/tmp.KfwiunHoeb /tmp/tmp.jzTy69OlGX
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.LD1fv5Or34
++ mktemp
+ local LAST_ERR=/tmp/tmp.6dsTILCfnu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LD1fv5Or34
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("8ae48b42-4fb4-40ce-823e-94bac6988aa5") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.6dsTILCfnu
+ rm /tmp/tmp.LD1fv5Or34 /tmp/tmp.6dsTILCfnu
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T19:53:56+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7cZn8MgvoO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lk7SaubU3j
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7cZn8MgvoO
++ cat /tmp/tmp.lk7SaubU3j
++ rm /tmp/tmp.7cZn8MgvoO /tmp/tmp.lk7SaubU3j
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.M4hFfAyJxL
++ mktemp
+ local LAST_ERR=/tmp/tmp.YLqKFSnx37
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.M4hFfAyJxL
+ cat /tmp/tmp.YLqKFSnx37
+ rm /tmp/tmp.M4hFfAyJxL /tmp/tmp.YLqKFSnx37
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.QSQ6VoS7pc/find-2nd
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-18T19:53:59+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Z3iGjqs51v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T3zKD79QtR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Z3iGjqs51v
++ cat /tmp/tmp.T3zKD79QtR
++ rm /tmp/tmp.Z3iGjqs51v /tmp/tmp.T3zKD79QtR
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qRR9sXwkZe
++ mktemp
+ local LAST_ERR=/tmp/tmp.VBt17An8ph
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qRR9sXwkZe
+ cat /tmp/tmp.VBt17An8ph
+ rm /tmp/tmp.qRR9sXwkZe /tmp/tmp.VBt17An8ph
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.QSQ6VoS7pc/find-2nd1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-18T19:54:01+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.j9EE4f8y0o
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3ypI4mWJmX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.j9EE4f8y0o
++ cat /tmp/tmp.3ypI4mWJmX
++ rm /tmp/tmp.j9EE4f8y0o /tmp/tmp.3ypI4mWJmX
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.PlUQGaqsbv
++ mktemp
+ local LAST_ERR=/tmp/tmp.7qdQtnr78B
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PlUQGaqsbv
+ cat /tmp/tmp.7qdQtnr78B
+ rm /tmp/tmp.PlUQGaqsbv /tmp/tmp.7qdQtnr78B
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.QSQ6VoS7pc/find-2nd2
+ run_restore backup-azure-blob
+ local backup_name=backup-azure-blob
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/restore.yml
+ log 'running restore restore-backup-azure-blob'
+ set +o xtrace
[2026-06-18T19:54:04+0000] running restore restore-backup-azure-blob
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.7iLRRrxSBT
++ mktemp
+ local LAST_ERR=/tmp/tmp.L3VrtjdtbU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7iLRRrxSBT
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
+ cat /tmp/tmp.L3VrtjdtbU
+ rm /tmp/tmp.7iLRRrxSBT /tmp/tmp.L3VrtjdtbU
+ return 0
+ wait_restore backup-azure-blob some-name
+ local backup_name=backup-azure-blob
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob object to be created.OK
Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jfyqHGSHyr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Pj51kOBXZb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jfyqHGSHyr
++ cat /tmp/tmp.Pj51kOBXZb
++ rm /tmp/tmp.jfyqHGSHyr /tmp/tmp.Pj51kOBXZb
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.imBSe5PVN5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dw2sG7oZcf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.imBSe5PVN5
++ cat /tmp/tmp.dw2sG7oZcf
++ rm /tmp/tmp.imBSe5PVN5 /tmp/tmp.dw2sG7oZcf
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hKDHZHsfei
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1QZdhteutG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hKDHZHsfei
++ cat /tmp/tmp.1QZdhteutG
++ rm /tmp/tmp.hKDHZHsfei /tmp/tmp.1QZdhteutG
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6KnKIJgAmn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rrFsHOFASO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6KnKIJgAmn
++ cat /tmp/tmp.rrFsHOFASO
++ rm /tmp/tmp.6KnKIJgAmn /tmp/tmp.rrFsHOFASO
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ check_data
+ local postfix=
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T19:56:12+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xh5LNomQcA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.avKWopl1FL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xh5LNomQcA
++ cat /tmp/tmp.avKWopl1FL
++ rm /tmp/tmp.xh5LNomQcA /tmp/tmp.avKWopl1FL
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.iLXtHRfgoK
++ mktemp
+ local LAST_ERR=/tmp/tmp.mtXdFFHTur
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iLXtHRfgoK
+ cat /tmp/tmp.mtXdFFHTur
+ rm /tmp/tmp.iLXtHRfgoK /tmp/tmp.mtXdFFHTur
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.QSQ6VoS7pc/find
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-18T19:56:14+0000] running db.test.command() in myApp1
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BtoFK0KZnX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.j0TUO64ZLr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BtoFK0KZnX
++ cat /tmp/tmp.j0TUO64ZLr
++ rm /tmp/tmp.BtoFK0KZnX /tmp/tmp.j0TUO64ZLr
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pYnk4pYN3G
++ mktemp
+ local LAST_ERR=/tmp/tmp.YzDi1Zfojo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pYnk4pYN3G
+ cat /tmp/tmp.YzDi1Zfojo
+ rm /tmp/tmp.pYnk4pYN3G /tmp/tmp.YzDi1Zfojo
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.QSQ6VoS7pc/find1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-18T19:56:16+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C4YJgAw537
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ljP2UmYDic
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C4YJgAw537
++ cat /tmp/tmp.ljP2UmYDic
++ rm /tmp/tmp.C4YJgAw537 /tmp/tmp.ljP2UmYDic
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.q9KkD8M0Uu
++ mktemp
+ local LAST_ERR=/tmp/tmp.MesYAwnefd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q9KkD8M0Uu
+ cat /tmp/tmp.MesYAwnefd
+ rm /tmp/tmp.q9KkD8M0Uu /tmp/tmp.MesYAwnefd
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.QSQ6VoS7pc/find2
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

+ check_backup_in_storage backup-minio minio rs0 myApp.test.gz
+ local backup=backup-minio
+ local storage_type=minio
+ local replset=rs0
+ local file=myApp.test.gz
+ local protocol=http
+ local endpoint
++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ sed -e 's/.json$//'
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kUKNtBnd25
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pQ2xnl7OcZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kUKNtBnd25
++ cat /tmp/tmp.pQ2xnl7OcZ
++ rm /tmp/tmp.kUKNtBnd25 /tmp/tmp.pQ2xnl7OcZ
++ return 0
+ backup_dest=operator-testing/2026-06-18T19:45:37Z
+ case ${storage_type} in
+ endpoint=minio-service
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-06-18T19:45:37Z/rs0/myApp.test.gz
+ grep myApp.test.gz
++ mktemp
+ local LAST_OUT=/tmp/tmp.xYOWt6E7WD
++ mktemp
+ local LAST_ERR=/tmp/tmp.34doH71XPZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-06-18T19:45:37Z/rs0/myApp.test.gz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xYOWt6E7WD
+ cat /tmp/tmp.34doH71XPZ
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-sharded-3020
+ rm /tmp/tmp.xYOWt6E7WD /tmp/tmp.34doH71XPZ
+ return 0
2026-06-18 19:45:42         55 myApp.test.gz
2026-06-18 19:45:42         55 myApp.test.gz
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YdRCPBmzw4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZispcMsewu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YdRCPBmzw4
++ cat /tmp/tmp.ZispcMsewu
++ rm /tmp/tmp.YdRCPBmzw4 /tmp/tmp.ZispcMsewu
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.avxXWAqOZw
++ mktemp
+ local LAST_ERR=/tmp/tmp.o04vfxCnlj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.avxXWAqOZw
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("60064e13-011f-467a-8f61-1e8aba98ddb3") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.o04vfxCnlj
+ rm /tmp/tmp.avxXWAqOZw /tmp/tmp.o04vfxCnlj
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cc6ieLvwKO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B6sr7FjvLy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cc6ieLvwKO
++ cat /tmp/tmp.B6sr7FjvLy
++ rm /tmp/tmp.cc6ieLvwKO /tmp/tmp.B6sr7FjvLy
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3MI2iC9Kt5
++ mktemp
+ local LAST_ERR=/tmp/tmp.pkAIjTaUDJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3MI2iC9Kt5
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("d02b29ce-1ba7-4a84-ad48-4d83801952d5") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.pkAIjTaUDJ
+ rm /tmp/tmp.3MI2iC9Kt5 /tmp/tmp.pkAIjTaUDJ
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gtXwOEGu49
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uDmUp0tOo4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gtXwOEGu49
++ cat /tmp/tmp.uDmUp0tOo4
++ rm /tmp/tmp.gtXwOEGu49 /tmp/tmp.uDmUp0tOo4
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.2tqkgCap8x
++ mktemp
+ local LAST_ERR=/tmp/tmp.VIKt1C4mGD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2tqkgCap8x
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("97c96864-d07e-4a6e-a5ad-dd7159190690") }
Percona Server for MongoDB server version: v8.0.23-10
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.VIKt1C4mGD
+ rm /tmp/tmp.2tqkgCap8x /tmp/tmp.VIKt1C4mGD
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T19:56:35+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VKp0buLIQL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IBQAFkZX1w
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VKp0buLIQL
++ cat /tmp/tmp.IBQAFkZX1w
++ rm /tmp/tmp.VKp0buLIQL /tmp/tmp.IBQAFkZX1w
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.EeeNn77Cdq
++ mktemp
+ local LAST_ERR=/tmp/tmp.gjyM4NcZqP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EeeNn77Cdq
+ cat /tmp/tmp.gjyM4NcZqP
+ rm /tmp/tmp.EeeNn77Cdq /tmp/tmp.gjyM4NcZqP
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.QSQ6VoS7pc/find-2nd
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-18T19:56:38+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CUUBk97tFq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JBodsNdEKt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CUUBk97tFq
++ cat /tmp/tmp.JBodsNdEKt
++ rm /tmp/tmp.CUUBk97tFq /tmp/tmp.JBodsNdEKt
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rkT5Wcdbtq
++ mktemp
+ local LAST_ERR=/tmp/tmp.jK42Dbjiq9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rkT5Wcdbtq
+ cat /tmp/tmp.jK42Dbjiq9
+ rm /tmp/tmp.rkT5Wcdbtq /tmp/tmp.jK42Dbjiq9
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.QSQ6VoS7pc/find-2nd1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-18T19:56:40+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S9lt3VlGBy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.icWVIrfMQB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.S9lt3VlGBy
++ cat /tmp/tmp.icWVIrfMQB
++ rm /tmp/tmp.S9lt3VlGBy /tmp/tmp.icWVIrfMQB
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8IqoPdK1yQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.q4YAJ0thEv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8IqoPdK1yQ
+ cat /tmp/tmp.q4YAJ0thEv
+ rm /tmp/tmp.8IqoPdK1yQ /tmp/tmp.q4YAJ0thEv
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.QSQ6VoS7pc/find-2nd2
+ run_restore backup-minio
+ local backup_name=backup-minio
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/restore.yml
+ log 'running restore restore-backup-minio'
+ set +o xtrace
[2026-06-18T19:56:42+0000] running restore restore-backup-minio
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.rzo4U0o4eR
++ mktemp
+ local LAST_ERR=/tmp/tmp.D9lviQRqIz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rzo4U0o4eR
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
+ cat /tmp/tmp.D9lviQRqIz
+ rm /tmp/tmp.rzo4U0o4eR /tmp/tmp.D9lviQRqIz
+ return 0
+ wait_restore backup-minio some-name
+ local backup_name=backup-minio
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio object to be created.OK
Waiting psmdb-restore/restore-backup-minio to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.v5LBVyFaHl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ApppvdZMeW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.v5LBVyFaHl
++ cat /tmp/tmp.ApppvdZMeW
++ rm /tmp/tmp.v5LBVyFaHl /tmp/tmp.ApppvdZMeW
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rbbB3gvNHy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GzaApr7GVl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rbbB3gvNHy
++ cat /tmp/tmp.GzaApr7GVl
++ rm /tmp/tmp.rbbB3gvNHy /tmp/tmp.GzaApr7GVl
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NSkSGEwG19
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qu2cCZkcdI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NSkSGEwG19
++ cat /tmp/tmp.qu2cCZkcdI
++ rm /tmp/tmp.NSkSGEwG19 /tmp/tmp.qu2cCZkcdI
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zppWIemYLe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IEQ084SHo8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zppWIemYLe
++ cat /tmp/tmp.IEQ084SHo8
++ rm /tmp/tmp.zppWIemYLe /tmp/tmp.IEQ084SHo8
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ check_data
+ local postfix=
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-06-18T19:58:45+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NwTk7QA0Oj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1cP8mNJu1w
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NwTk7QA0Oj
++ cat /tmp/tmp.1cP8mNJu1w
++ rm /tmp/tmp.NwTk7QA0Oj /tmp/tmp.1cP8mNJu1w
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Mwu2oCu3ec
++ mktemp
+ local LAST_ERR=/tmp/tmp.mn7OCK9qBX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Mwu2oCu3ec
+ cat /tmp/tmp.mn7OCK9qBX
+ rm /tmp/tmp.Mwu2oCu3ec /tmp/tmp.mn7OCK9qBX
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.QSQ6VoS7pc/find
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp1'
+ set +o xtrace
[2026-06-18T19:58:48+0000] running db.test.command() in myApp1
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lOTOfFMXG8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FrT7VyGElR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lOTOfFMXG8
++ cat /tmp/tmp.FrT7VyGElR
++ rm /tmp/tmp.lOTOfFMXG8 /tmp/tmp.FrT7VyGElR
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OculehMaSi
++ mktemp
+ local LAST_ERR=/tmp/tmp.KG1VjhMCnu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OculehMaSi
+ cat /tmp/tmp.KG1VjhMCnu
+ rm /tmp/tmp.OculehMaSi /tmp/tmp.KG1VjhMCnu
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.QSQ6VoS7pc/find1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-3020 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp2'
+ set +o xtrace
[2026-06-18T19:58:51+0000] running db.test.command() in myApp2
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-3020 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-3020
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.V1sogoxifw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3JfAS16OvO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.V1sogoxifw
++ cat /tmp/tmp.3JfAS16OvO
++ rm /tmp/tmp.V1sogoxifw /tmp/tmp.3JfAS16OvO
++ return 0
+ local client_container=psmdb-client-bb8b97679-2s56p
+ kubectl_bin exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DngRRFcAvw
++ mktemp
+ local LAST_ERR=/tmp/tmp.0oWD7jlwbd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-2s56p -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-3020.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DngRRFcAvw
+ cat /tmp/tmp.0oWD7jlwbd
+ rm /tmp/tmp.DngRRFcAvw /tmp/tmp.0oWD7jlwbd
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.QSQ6VoS7pc/find2
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.yxBRprys33
++ mktemp
+ local LAST_ERR=/tmp/tmp.IjIDg6jYJw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yxBRprys33
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-sharded-3020 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-sharded-3020 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs" deleted from demand-backup-sharded-3020 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-sharded-3020 namespace
+ cat /tmp/tmp.IjIDg6jYJw
+ rm /tmp/tmp.yxBRprys33 /tmp/tmp.IjIDg6jYJw
+ return 0
++ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ grep -c _
++ cat
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iGrVvY9zol
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ohSJhpYX6G
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iGrVvY9zol
++ cat /tmp/tmp.ohSJhpYX6G
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-sharded-3020
++ rm /tmp/tmp.iGrVvY9zol /tmp/tmp.ohSJhpYX6G
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion_aws operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup-sharded/2026-06-18T19:47:19Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded/2026-06-18T19:47:19Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup-sharded/2026-06-18T19:47:19Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3'
Backup psmdb-demand-backup-sharded/2026-06-18T19:47:19Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:45Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_azure operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup-sharded/2026-06-18T19:46:10Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob'
Backup psmdb-demand-backup-sharded/2026-06-18T19:46:10Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/container-rc.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.DUCZZdaKlh
++ mktemp
+ local LAST_ERR=/tmp/tmp.oJY52UKkab
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/e2e-tests/conf/container-rc.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DUCZZdaKlh
runtimeclass.node.k8s.io "container-rc" deleted
+ cat /tmp/tmp.oJY52UKkab
+ rm /tmp/tmp.DUCZZdaKlh /tmp/tmp.oJY52UKkab
+ return 0
+ destroy demand-backup-sharded-3020
+ local namespace=demand-backup-sharded-3020
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ wc -l
++ kubectl_bin get psmdb-backup --no-headers
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3K26SjaDx8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Qrd9yVccT5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3K26SjaDx8
++ cat /tmp/tmp.Qrd9yVccT5
No resources found in demand-backup-sharded-3020 namespace.
++ rm /tmp/tmp.3K26SjaDx8 /tmp/tmp.Qrd9yVccT5
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.7Pj1HyzhZh
++ mktemp
+ local LAST_ERR=/tmp/tmp.knj6YjgGNx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7Pj1HyzhZh
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.knj6YjgGNx
+ rm /tmp/tmp.7Pj1HyzhZh /tmp/tmp.knj6YjgGNx
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ua1xvJ100a
++ mktemp
+ local LAST_ERR=/tmp/tmp.Sle2AmQkqw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ua1xvJ100a
+ cat /tmp/tmp.Sle2AmQkqw
+ rm /tmp/tmp.Ua1xvJ100a /tmp/tmp.Sle2AmQkqw
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.JsHrKVuBhC
++ mktemp
+ local LAST_ERR=/tmp/tmp.E6l4T3U0V3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JsHrKVuBhC
+ cat /tmp/tmp.E6l4T3U0V3
+ rm /tmp/tmp.JsHrKVuBhC /tmp/tmp.E6l4T3U0V3
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.UfViIrzgRl
++ mktemp
+ local LAST_ERR=/tmp/tmp.cIDAE6h95t
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UfViIrzgRl
+ cat /tmp/tmp.cIDAE6h95t
+ rm /tmp/tmp.UfViIrzgRl /tmp/tmp.cIDAE6h95t
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.IlQXuvNNxi
++ mktemp
+ local LAST_ERR=/tmp/tmp.lspysTR1Yd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IlQXuvNNxi
+ cat /tmp/tmp.lspysTR1Yd
+ rm /tmp/tmp.IlQXuvNNxi /tmp/tmp.lspysTR1Yd
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.NLcurHJ5TU
++ mktemp
+ local LAST_ERR=/tmp/tmp.cT4BrC2y8U
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2406/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NLcurHJ5TU
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.cT4BrC2y8U
+ rm /tmp/tmp.NLcurHJ5TU /tmp/tmp.cT4BrC2y8U
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.AXnWrYK5OA
++ mktemp
+ local LAST_ERR=/tmp/tmp.o8jDu84UVW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.AXnWrYK5OA
+ cat /tmp/tmp.o8jDu84UVW
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.AXnWrYK5OA
+ cat /tmp/tmp.o8jDu84UVW
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.AXnWrYK5OA
+ cat /tmp/tmp.o8jDu84UVW
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.AXnWrYK5OA
+ cat /tmp/tmp.o8jDu84UVW
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.AXnWrYK5OA /tmp/tmp.o8jDu84UVW
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-sharded-3020
+ rm -rf /tmp/tmp.QSQ6VoS7pc
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.lBf1vx9LzV
+ local LAST_OUT=/tmp/tmp.DSs0EGJ93M
++ mktemp
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_ERR=/tmp/tmp.34Id6qXMPZ
+ local LAST_ERR=/tmp/tmp.Wj3DzKzVRj
+ local exit_status=0
+ local exit_status=0
+ local timeout=4
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-sharded-3020
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator