++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/logs/demand-backup-sharded.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/logs/demand-backup-sharded.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ kubectl get nodes
++ grep '^minikube'
+++ jq -r .serverVersion.gitVersion
+++ kubectl version -o json
+++ grep '\-eks\-'
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
++ '[' ']'
++ EKS=0
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep gke
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
++ '[' v1.30.12-gke.1414000 ']'
++ GKE=1
+++ kubectl version -o json
+++ /usr/bin/sed -r 's/[^0-9.]+//g'
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.30
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ custom_port=27019
+ create_infra demand-backup-sharded-19519
+ local ns=demand-backup-sharded-19519
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.T5He5Pywaq
++ mktemp
+ local LAST_ERR=/tmp/tmp.H0ism7GhZu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.T5He5Pywaq
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.H0ism7GhZu
+ rm /tmp/tmp.T5He5Pywaq /tmp/tmp.H0ism7GhZu
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.ceTF2GrXuH
++ mktemp
+ local LAST_ERR=/tmp/tmp.LYzrgeFJ1e
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ceTF2GrXuH
+ cat /tmp/tmp.LYzrgeFJ1e
+ rm /tmp/tmp.ceTF2GrXuH /tmp/tmp.LYzrgeFJ1e
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.nD997VngmP
++ mktemp
+ local LAST_ERR=/tmp/tmp.6YMAvoOeTZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nD997VngmP
+ cat /tmp/tmp.6YMAvoOeTZ
+ rm /tmp/tmp.nD997VngmP /tmp/tmp.6YMAvoOeTZ
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.I5TEjb87VJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.MP7HEHOFVn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I5TEjb87VJ
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met
+ cat /tmp/tmp.MP7HEHOFVn
+ rm /tmp/tmp.I5TEjb87VJ /tmp/tmp.MP7HEHOFVn
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.yEphmDSxtp
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZOZsmVn1zc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yEphmDSxtp
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.ZOZsmVn1zc
+ rm /tmp/tmp.yEphmDSxtp /tmp/tmp.ZOZsmVn1zc
+ return 0
+ check_crd_for_deletion PR-1961-970eeaa4
+ local git_tag=PR-1961-970eeaa4
++ yq eval .metadata.name
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1961-970eeaa4/deploy/crd.yaml
++ /usr/bin/sed s/---//g
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dtk509yogk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B7BXEnaFio
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.dtk509yogk
++ cat /tmp/tmp.B7BXEnaFio
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.dtk509yogk
++ cat /tmp/tmp.B7BXEnaFio
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.dtk509yogk
++ cat /tmp/tmp.B7BXEnaFio
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.dtk509yogk
++ cat /tmp/tmp.B7BXEnaFio
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.dtk509yogk /tmp/tmp.B7BXEnaFio
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

++ mktemp
+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.qTqHjTrl2j
++ mktemp
+ local LAST_ERR=/tmp/tmp.rk5zLYfyt9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.YLWicm8qaP
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ local LAST_ERR=/tmp/tmp.GaSvVbJaBG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YLWicm8qaP
+ cat /tmp/tmp.GaSvVbJaBG
+ rm /tmp/tmp.YLWicm8qaP /tmp/tmp.GaSvVbJaBG
+ return 0
namespace "demand-backup-sharded-12495" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qTqHjTrl2j
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.rk5zLYfyt9
+ rm /tmp/tmp.qTqHjTrl2j /tmp/tmp.rk5zLYfyt9
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.nwxYgZTP0D
++ mktemp
+ local LAST_ERR=/tmp/tmp.Z0CIsCKvPI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nwxYgZTP0D
+ cat /tmp/tmp.Z0CIsCKvPI
+ rm /tmp/tmp.nwxYgZTP0D /tmp/tmp.Z0CIsCKvPI
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.T51ROSi1AG
++ mktemp
+ local LAST_ERR=/tmp/tmp.tM72PAEYLK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.T51ROSi1AG
namespace/psmdb-operator created
+ cat /tmp/tmp.tM72PAEYLK
+ rm /tmp/tmp.T51ROSi1AG /tmp/tmp.tM72PAEYLK
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SQxVDlG03W
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uLr8nliVi1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SQxVDlG03W
++ cat /tmp/tmp.uLr8nliVi1
++ rm /tmp/tmp.SQxVDlG03W /tmp/tmp.uLr8nliVi1
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-970eeaa4-5-cluster8 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.K43e1q3PZN
++ mktemp
+ local LAST_ERR=/tmp/tmp.xzMu4ekin0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-970eeaa4-5-cluster8 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.K43e1q3PZN
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-970eeaa4-5-cluster8" modified.
+ cat /tmp/tmp.xzMu4ekin0
+ rm /tmp/tmp.K43e1q3PZN /tmp/tmp.xzMu4ekin0
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.iCKdCDNrTB
++ mktemp
+ local LAST_ERR=/tmp/tmp.y3tnwdojkV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iCKdCDNrTB
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.y3tnwdojkV
+ rm /tmp/tmp.iCKdCDNrTB /tmp/tmp.y3tnwdojkV
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.5rsNKzxtbX
++ mktemp
+ local LAST_ERR=/tmp/tmp.IO4zLZjxES
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5rsNKzxtbX
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.IO4zLZjxES
+ rm /tmp/tmp.5rsNKzxtbX /tmp/tmp.IO4zLZjxES
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1961-970eeaa4") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
+ kubectl_bin apply -f -
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.U9Kqeh6ubj
++ mktemp
+ local LAST_ERR=/tmp/tmp.dofpS3zHSd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.U9Kqeh6ubj
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.dofpS3zHSd
+ rm /tmp/tmp.U9Kqeh6ubj /tmp/tmp.dofpS3zHSd
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JtodmCje9z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xgF7BO8yPt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JtodmCje9z
++ cat /tmp/tmp.xgF7BO8yPt
++ rm /tmp/tmp.JtodmCje9z /tmp/tmp.xgF7BO8yPt
++ return 0
+ wait_pod percona-server-mongodb-operator-7cfd5675c9-dl6cx
+ local pod=percona-server-mongodb-operator-7cfd5675c9-dl6cx
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-7cfd5675c9-dl6cx to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QiUE9CGxuS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Jdgxp0HF42
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QiUE9CGxuS
++ cat /tmp/tmp.Jdgxp0HF42
++ rm /tmp/tmp.QiUE9CGxuS /tmp/tmp.Jdgxp0HF42
++ return 0
+ kubectl_bin logs percona-server-mongodb-operator-7cfd5675c9-dl6cx
++ mktemp
+ local LAST_OUT=/tmp/tmp.S1hvP3gFpJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.nI2MI42JhF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs percona-server-mongodb-operator-7cfd5675c9-dl6cx
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.S1hvP3gFpJ
+ cat /tmp/tmp.nI2MI42JhF
+ rm /tmp/tmp.S1hvP3gFpJ /tmp/tmp.nI2MI42JhF
+ return 0
2025-08-06T20:55:32.509Z	INFO	setup	Manager starting up	{"gitCommit": "970eeaa4b725700bdc63e3d7ec04ab0be1ecce33", "gitBranch": "PR-1961-970eeaa4", "buildTime": "", "goVersion": "go1.24.5", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-sharded-19519
+ local namespace=demand-backup-sharded-19519
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-sharded-19519'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-sharded-19519
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-sharded-19519 --ignore-not-found
+ awk '{print$1}'
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.FmBgX36fSo
++ mktemp
+ local LAST_OUT=/tmp/tmp.D4GBPGUIf7
++ mktemp
+ local LAST_ERR=/tmp/tmp.rpea1goKoI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.4X7LvXfygl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-sharded-19519 --ignore-not-found
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.D4GBPGUIf7
+ cat /tmp/tmp.4X7LvXfygl
+ rm /tmp/tmp.D4GBPGUIf7 /tmp/tmp.4X7LvXfygl
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FmBgX36fSo
+ cat /tmp/tmp.rpea1goKoI
+ rm /tmp/tmp.FmBgX36fSo /tmp/tmp.rpea1goKoI
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-sharded-19519
++ mktemp
+ local LAST_OUT=/tmp/tmp.t1aCovHh78
++ mktemp
+ local LAST_ERR=/tmp/tmp.8vhJm4Wm7N
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace demand-backup-sharded-19519
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t1aCovHh78
+ cat /tmp/tmp.8vhJm4Wm7N
+ rm /tmp/tmp.t1aCovHh78 /tmp/tmp.8vhJm4Wm7N
+ return 0
+ desc 'create namespace demand-backup-sharded-19519'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-sharded-19519
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-sharded-19519
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ka0AYhvVHZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.psbCBdFFlt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace demand-backup-sharded-19519
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ka0AYhvVHZ
namespace/demand-backup-sharded-19519 created
+ cat /tmp/tmp.psbCBdFFlt
+ rm /tmp/tmp.Ka0AYhvVHZ /tmp/tmp.psbCBdFFlt
+ return 0
+ set_kube_ctx demand-backup-sharded-19519
+ local namespace=demand-backup-sharded-19519
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hoRmfBbHFc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3aOKjcFkWG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hoRmfBbHFc
++ cat /tmp/tmp.3aOKjcFkWG
++ rm /tmp/tmp.hoRmfBbHFc /tmp/tmp.3aOKjcFkWG
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-970eeaa4-5-cluster8 --namespace=demand-backup-sharded-19519
++ mktemp
+ local LAST_OUT=/tmp/tmp.FP2njCjIwe
++ mktemp
+ local LAST_ERR=/tmp/tmp.2dNTTqseDC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-970eeaa4-5-cluster8 --namespace=demand-backup-sharded-19519
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FP2njCjIwe
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-970eeaa4-5-cluster8" modified.
+ cat /tmp/tmp.2dNTTqseDC
+ rm /tmp/tmp.FP2njCjIwe /tmp/tmp.2dNTTqseDC
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Wed Aug  6 20:55:47 2025
NAMESPACE: demand-backup-sharded-19519
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-sharded-19519.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-19519 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-19519

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-sharded-19519 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-sharded-19519 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E9x2tUPhwT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fUFhf91hr1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E9x2tUPhwT
++ cat /tmp/tmp.fUFhf91hr1
++ rm /tmp/tmp.E9x2tUPhwT /tmp/tmp.fUFhf91hr1
++ return 0
+ MINIO_POD=minio-service-86dfccd949-6m9kd
+ wait_pod minio-service-86dfccd949-6m9kd
+ local pod=minio-service-86dfccd949-6m9kd
+ set +o xtrace
waiting for pod/minio-service-86dfccd949-6m9kd to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-sharded-19519.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.CKWb1Zg5RC
++ mktemp
+ local LAST_ERR=/tmp/tmp.GYd1bRm1xV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-sharded-19519.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CKWb1Zg5RC
service/minio-service created
+ cat /tmp/tmp.GYd1bRm1xV
+ rm /tmp/tmp.CKWb1Zg5RC /tmp/tmp.GYd1bRm1xV
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.kpMZ5cZGxl
++ mktemp
+ local LAST_ERR=/tmp/tmp.D3C7TcNx3s
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kpMZ5cZGxl
pod "aws-cli" deleted
+ cat /tmp/tmp.D3C7TcNx3s
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.kpMZ5cZGxl /tmp/tmp.D3C7TcNx3s
+ return 0
+ desc 'create first PSMDB cluster with custom port 27019'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster with custom port 27019
-----------------------------------------------------------------------------------

+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.xeYEedLIXf
++ mktemp
+ local LAST_ERR=/tmp/tmp.RdsDwMp5K8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xeYEedLIXf
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.RdsDwMp5K8
+ rm /tmp/tmp.xeYEedLIXf /tmp/tmp.RdsDwMp5K8
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.evESwZKtwH
++ mktemp
+ local LAST_ERR=/tmp/tmp.KdYTiIkVK9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.evESwZKtwH
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.KdYTiIkVK9
+ rm /tmp/tmp.evESwZKtwH /tmp/tmp.KdYTiIkVK9
+ return 0
+ version_gt 1.19
++ echo '1.30 >= 1.19'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' 0 -ne 1 ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/container-rc.yaml
+ /usr/bin/sed s/docker/runc/g
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.KAdFaeaMZw
++ mktemp
+ local LAST_ERR=/tmp/tmp.PMKV8isZc0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KAdFaeaMZw
runtimeclass.node.k8s.io/container-rc unchanged
+ cat /tmp/tmp.PMKV8isZc0
+ rm /tmp/tmp.KAdFaeaMZw /tmp/tmp.PMKV8isZc0
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1961-970eeaa4"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.QvGuVfJBgt
++ mktemp
+ local LAST_ERR=/tmp/tmp.upEBq2NpHL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QvGuVfJBgt
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.upEBq2NpHL
+ rm /tmp/tmp.QvGuVfJBgt /tmp/tmp.upEBq2NpHL
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rGPNZIo5mh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZPru6c9izU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rGPNZIo5mh
++ cat /tmp/tmp.ZPru6c9izU
++ rm /tmp/tmp.rGPNZIo5mh /tmp/tmp.ZPru6c9izU
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eXOUGM5QK2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WyERGxKEyq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eXOUGM5QK2
++ cat /tmp/tmp.WyERGxKEyq
++ rm /tmp/tmp.eXOUGM5QK2 /tmp/tmp.WyERGxKEyq
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lNjJPjmMEf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mW06Tov0Lt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lNjJPjmMEf
++ cat /tmp/tmp.mW06Tov0Lt
++ rm /tmp/tmp.lNjJPjmMEf /tmp/tmp.mW06Tov0Lt
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness........................................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3eznzD7xV2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s3holnU1zd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3eznzD7xV2
++ cat /tmp/tmp.s3holnU1zd
++ rm /tmp/tmp.3eznzD7xV2 /tmp/tmp.s3holnU1zd
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EoC49ZsCw3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3aIAZQDz4k
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EoC49ZsCw3
++ cat /tmp/tmp.3aIAZQDz4k
++ rm /tmp/tmp.EoC49ZsCw3 /tmp/tmp.3aIAZQDz4k
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zM4CFIaHOC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3AlJie0yey
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zM4CFIaHOC
++ cat /tmp/tmp.3AlJie0yey
++ rm /tmp/tmp.zM4CFIaHOC /tmp/tmp.3AlJie0yey
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vhAbGGfSp9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uaIGb2x5LN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vhAbGGfSp9
++ cat /tmp/tmp.uaIGb2x5LN
++ rm /tmp/tmp.vhAbGGfSp9 /tmp/tmp.uaIGb2x5LN
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jwOdYFUZFL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nFgNxqnK1F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jwOdYFUZFL
++ cat /tmp/tmp.nFgNxqnK1F
++ rm /tmp/tmp.jwOdYFUZFL /tmp/tmp.nFgNxqnK1F
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1MkevM9JMQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BQKLFSKx2l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1MkevM9JMQ
++ cat /tmp/tmp.BQKLFSKx2l
++ rm /tmp/tmp.1MkevM9JMQ /tmp/tmp.BQKLFSKx2l
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-19519", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZP3w6ql3aW
++ mktemp
+ local LAST_ERR=/tmp/tmp.4XRojxJACk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZP3w6ql3aW
+ cat /tmp/tmp.4XRojxJACk
+ rm /tmp/tmp.ZP3w6ql3aW /tmp/tmp.4XRojxJACk
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ bc -l
++ echo '1.30 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml /tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs0.yml
+ compare_kubectl statefulset/some-name-rs1
+ local resource=statefulset/some-name-rs1
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml
+ local new_result=/tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs1.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs1
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-19519", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ATelfxxooB
++ mktemp
+ local LAST_ERR=/tmp/tmp.918Ie6KnKX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ATelfxxooB
+ cat /tmp/tmp.918Ie6KnKX
+ rm /tmp/tmp.ATelfxxooB /tmp/tmp.918Ie6KnKX
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs1.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs1.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs1.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml /tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs1.yml
+ compare_kubectl statefulset/some-name-rs2
+ local resource=statefulset/some-name-rs2
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml
+ local new_result=/tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs2.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs2
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-19519", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.lktMf1Ualz
++ mktemp
+ local LAST_ERR=/tmp/tmp.GsOXbxMTer
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs2
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lktMf1Ualz
+ cat /tmp/tmp.GsOXbxMTer
+ rm /tmp/tmp.lktMf1Ualz /tmp/tmp.GsOXbxMTer
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs2.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs2.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs2.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml /tmp/tmp.94rJ6ctuxD/statefulset_some-name-rs2.yml
+ compare_kubectl statefulset/some-name-cfg
+ local resource=statefulset/some-name-cfg
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml
+ local new_result=/tmp/tmp.94rJ6ctuxD/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-19519", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xwI9HXTjuZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.CvTbaZJvTj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xwI9HXTjuZ
+ cat /tmp/tmp.CvTbaZJvTj
+ rm /tmp/tmp.xwI9HXTjuZ /tmp/tmp.CvTbaZJvTj
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml /tmp/tmp.94rJ6ctuxD/statefulset_some-name-cfg.yml
+ compare_kubectl statefulset/some-name-mongos ''
+ local resource=statefulset/some-name-mongos
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml
+ local new_result=/tmp/tmp.94rJ6ctuxD/statefulset_some-name-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-mongos
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-19519", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.5NBpiEK8vD
++ mktemp
+ local LAST_ERR=/tmp/tmp.DGcuwxwbSh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5NBpiEK8vD
+ cat /tmp/tmp.DGcuwxwbSh
+ rm /tmp/tmp.5NBpiEK8vD /tmp/tmp.DGcuwxwbSh
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-mongos.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml /tmp/tmp.94rJ6ctuxD/statefulset_some-name-mongos.yml
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.mfRYo1XcWP
++ mktemp
+ local LAST_ERR=/tmp/tmp.L4DdSxgCti
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mfRYo1XcWP
secret/some-name-mongos created
+ cat /tmp/tmp.L4DdSxgCti
+ rm /tmp/tmp.mfRYo1XcWP /tmp/tmp.L4DdSxgCti
+ return 0
+ sleep 10
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.i5sXh30zie
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JZ5SYTEbdm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.i5sXh30zie
++ cat /tmp/tmp.JZ5SYTEbdm
++ rm /tmp/tmp.i5sXh30zie /tmp/tmp.JZ5SYTEbdm
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.elDslO4RFb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tij0jm6i2c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.elDslO4RFb
++ cat /tmp/tmp.tij0jm6i2c
++ rm /tmp/tmp.elDslO4RFb /tmp/tmp.tij0jm6i2c
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TFg7bWpGMw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jBvK9C3TYe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TFg7bWpGMw
++ cat /tmp/tmp.jBvK9C3TYe
++ rm /tmp/tmp.TFg7bWpGMw /tmp/tmp.jBvK9C3TYe
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.............................................................................................................
+ compare_kubectl statefulset/some-name-mongos -secret
+ local resource=statefulset/some-name-mongos
+ local postfix=-secret
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml
+ local new_result=/tmp/tmp.94rJ6ctuxD/statefulset_some-name-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-mongos
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-19519", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fgjcwmsMcf
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xa5MaxMLdA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fgjcwmsMcf
+ cat /tmp/tmp.Xa5MaxMLdA
+ rm /tmp/tmp.fgjcwmsMcf /tmp/tmp.Xa5MaxMLdA
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-mongos.yml
+ version_gt 1.22
++ bc -l
++ echo '1.30 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.94rJ6ctuxD/statefulset_some-name-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml /tmp/tmp.94rJ6ctuxD/statefulset_some-name-mongos.yml
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ArCQsUWtAI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jBILK4qskd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ArCQsUWtAI
++ cat /tmp/tmp.jBILK4qskd
++ rm /tmp/tmp.ArCQsUWtAI /tmp/tmp.jBILK4qskd
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kXxzXz359r
++ mktemp
+ local LAST_ERR=/tmp/tmp.N4XmI1y328
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kXxzXz359r
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("4fdb6af1-0efa-4474-bb44-35b3e3f9e0c3") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		},
		{
			"db" : "myApp1",
			"role" : "readWrite"
		},
		{
			"db" : "myApp2",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.N4XmI1y328
+ rm /tmp/tmp.kXxzXz359r /tmp/tmp.N4XmI1y328
+ return 0
+ run_mongos 'sh.enableSharding("myApp","rs0")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=sh.enableSharding("myApp","rs0")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CEpZNjnic2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9ZNoH9oiBe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CEpZNjnic2
++ cat /tmp/tmp.9ZNoH9oiBe
++ rm /tmp/tmp.CEpZNjnic2 /tmp/tmp.9ZNoH9oiBe
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Hd7i1n2CzY
++ mktemp
+ local LAST_ERR=/tmp/tmp.1g6n1HGSyW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Hd7i1n2CzY
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("a9169f90-bbdd-49f0-8d02-db65a7898289") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1754514274, 8),
		"signature" : {
			"hash" : BinData(0,"6f9+yUKpxt2nv+kMTGXTLHsNba0="),
			"keyId" : NumberLong("7535579709208264728")
		}
	},
	"operationTime" : Timestamp(1754514274, 2)
}
bye
+ cat /tmp/tmp.1g6n1HGSyW
+ rm /tmp/tmp.Hd7i1n2CzY /tmp/tmp.1g6n1HGSyW
+ return 0
+ run_mongos 'sh.enableSharding("myApp1","rs1")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=sh.enableSharding("myApp1","rs1")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GDTxmzASbx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BtAvRpuIJa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GDTxmzASbx
++ cat /tmp/tmp.BtAvRpuIJa
++ rm /tmp/tmp.GDTxmzASbx /tmp/tmp.BtAvRpuIJa
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.gRQlMvofra
++ mktemp
+ local LAST_ERR=/tmp/tmp.kmMdECs3Ve
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gRQlMvofra
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("29188e12-db10-4b72-bbc7-1c9f9d6c8fb9") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1754514276, 8),
		"signature" : {
			"hash" : BinData(0,"8VOnxVSPzjljSk2jKtJ76OOCPjg="),
			"keyId" : NumberLong("7535579709208264728")
		}
	},
	"operationTime" : Timestamp(1754514276, 3)
}
bye
+ cat /tmp/tmp.kmMdECs3Ve
+ rm /tmp/tmp.gRQlMvofra /tmp/tmp.kmMdECs3Ve
+ return 0
+ run_mongos 'sh.enableSharding("myApp2","rs2")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=sh.enableSharding("myApp2","rs2")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3sJaW4cfq6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oYnNtHpqEg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3sJaW4cfq6
++ cat /tmp/tmp.oYnNtHpqEg
++ rm /tmp/tmp.3sJaW4cfq6 /tmp/tmp.oYnNtHpqEg
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jwpbqCZRc7
++ mktemp
+ local LAST_ERR=/tmp/tmp.GQdRodXhSy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jwpbqCZRc7
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("2fd16260-5e97-4262-943b-98443bfad0bc") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1754514278, 7),
		"signature" : {
			"hash" : BinData(0,"6fJAIpOxVCu8bvuL9GeiCURntRc="),
			"keyId" : NumberLong("7535579709208264728")
		}
	},
	"operationTime" : Timestamp(1754514278, 2)
}
bye
+ cat /tmp/tmp.GQdRodXhSy
+ rm /tmp/tmp.jwpbqCZRc7 /tmp/tmp.GQdRodXhSy
+ return 0
+ insert_data_mongos 100500 myApp '' 27019
+ local data=100500
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VxTlt1VHtf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tRKOSXvmO0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VxTlt1VHtf
++ cat /tmp/tmp.tRKOSXvmO0
++ rm /tmp/tmp.VxTlt1VHtf /tmp/tmp.tRKOSXvmO0
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.epktMf8etm
++ mktemp
+ local LAST_ERR=/tmp/tmp.YBUTJIx2cm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.epktMf8etm
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("3c9fe845-42c1-4d1f-9eeb-5641e66200fa") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.YBUTJIx2cm
+ rm /tmp/tmp.epktMf8etm /tmp/tmp.YBUTJIx2cm
+ return 0
+ insert_data_mongos 100500 myApp1 '' 27019
+ local data=100500
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qq8t8P9W4s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JZQ055S1W3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qq8t8P9W4s
++ cat /tmp/tmp.JZQ055S1W3
++ rm /tmp/tmp.qq8t8P9W4s /tmp/tmp.JZQ055S1W3
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5SPev35k3g
++ mktemp
+ local LAST_ERR=/tmp/tmp.DIzMAjnIzj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5SPev35k3g
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("95ca6f94-3747-495b-b9c6-c622c5494535") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.DIzMAjnIzj
+ rm /tmp/tmp.5SPev35k3g /tmp/tmp.DIzMAjnIzj
+ return 0
+ insert_data_mongos 100500 myApp2 '' 27019
+ local data=100500
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JlqjWsH1bX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZWcxbLLl7S
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JlqjWsH1bX
++ cat /tmp/tmp.ZWcxbLLl7S
++ rm /tmp/tmp.JlqjWsH1bX /tmp/tmp.ZWcxbLLl7S
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.AqwrKEG9Sq
++ mktemp
+ local LAST_ERR=/tmp/tmp.K8S7aHiU4S
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AqwrKEG9Sq
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b1c8201d-1584-43cb-b372-0e6b8e6da2d5") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.K8S7aHiU4S
+ rm /tmp/tmp.AqwrKEG9Sq /tmp/tmp.K8S7aHiU4S
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' '' 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb '' '' 27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B7ve3Jk0Es
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZpqFB6ZNJn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.B7ve3Jk0Es
++ cat /tmp/tmp.ZpqFB6ZNJn
++ rm /tmp/tmp.B7ve3Jk0Es /tmp/tmp.ZpqFB6ZNJn
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.GDWqX6S6u0
++ mktemp
+ local LAST_ERR=/tmp/tmp.XqIs4SZvhu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GDWqX6S6u0
+ cat /tmp/tmp.XqIs4SZvhu
+ rm /tmp/tmp.GDWqX6S6u0 /tmp/tmp.XqIs4SZvhu
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.94rJ6ctuxD/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-08-06T20:59:56.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-08-06T20:59:59.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-08-06T20:59:55.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-0
+ local agent_pod=some-name-rs1-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-0...2025-08-06T20:59:59.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-1
+ local agent_pod=some-name-rs1-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-1...2025-08-06T20:59:59.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-2
+ local agent_pod=some-name-rs1-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-2...2025-08-06T20:59:55.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-0
+ local agent_pod=some-name-rs2-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-0...2025-08-06T21:00:01.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-1
+ local agent_pod=some-name-rs2-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-1...2025-08-06T21:00:01.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-2
+ local agent_pod=some-name-rs2-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-2...2025-08-06T21:00:01.000+0000 I listening for the commands

+ backup_name_aws=backup-aws-s3
+ backup_name_minio=backup-minio
+ backup_name_gcp=backup-gcp-cs
+ backup_name_azure=backup-azure-blob
+ desc 'run backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------

+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ desc 'run backup backup-minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-minio"
			| .spec.storageName = "minio"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.5sAuoGCZuP
++ mktemp
+ local LAST_ERR=/tmp/tmp.WsfZm2y94J
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5sAuoGCZuP
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.WsfZm2y94J
+ rm /tmp/tmp.5sAuoGCZuP /tmp/tmp.WsfZm2y94J
+ return 0
+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ desc 'run backup backup-aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-aws-s3
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-aws-s3"
			| .spec.storageName = "aws-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.nU9FKPhJzI
++ mktemp
+ local LAST_ERR=/tmp/tmp.fOPbhiYNIM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nU9FKPhJzI
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.fOPbhiYNIM
+ rm /tmp/tmp.nU9FKPhJzI /tmp/tmp.fOPbhiYNIM
+ return 0
+ run_backup gcp-cs
+ local storage=gcp-cs
+ local backup_name=backup-gcp-cs
+ local type=logical
+ desc 'run backup backup-gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-gcp-cs
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-gcp-cs"
			| .spec.storageName = "gcp-cs"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/backup-gcp-cs.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.VZWchJZf4v
++ mktemp
+ local LAST_ERR=/tmp/tmp.tgDS1kk1hf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VZWchJZf4v
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created
+ cat /tmp/tmp.tgDS1kk1hf
+ rm /tmp/tmp.VZWchJZf4v /tmp/tmp.tgDS1kk1hf
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ desc 'run backup backup-azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-azure-blob
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f -
+ yq eval '.metadata.name = "backup-azure-blob"
			| .spec.storageName = "azure-blob"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/backup-azure-blob.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.efereiZHld
++ mktemp
+ local LAST_ERR=/tmp/tmp.VLMkWKq6ul
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.efereiZHld
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.VLMkWKq6ul
+ rm /tmp/tmp.efereiZHld /tmp/tmp.VLMkWKq6ul
+ return 0
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state.................................
+ wait_backup backup-gcp-cs
+ local backup_name=backup-gcp-cs
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs to reach ready state...................
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state...................
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state.
+ sleep 5
+ '[' -z '' ']'
+ desc 'check backup and restore -- aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2nWqPMIyXS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H7qsU9zFSF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2nWqPMIyXS
++ cat /tmp/tmp.H7qsU9zFSF
++ rm /tmp/tmp.2nWqPMIyXS /tmp/tmp.H7qsU9zFSF
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:05:27Z
+ curl -s https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:05:27Z/rs0/myApp.test.gz
+ gunzip
+ gunzip
+ curl -s https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:05:27Z/rs1/myApp1.test.gz
+ curl -s https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:05:27Z/rs2/myApp2.test.gz
+ gunzip
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AsWrlKIpFz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ggxhnh8o3f
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AsWrlKIpFz
++ cat /tmp/tmp.Ggxhnh8o3f
++ rm /tmp/tmp.AsWrlKIpFz /tmp/tmp.Ggxhnh8o3f
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.heus4y31YE
++ mktemp
+ local LAST_ERR=/tmp/tmp.hd7JsLesT5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.heus4y31YE
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("1f0ee838-117c-475b-a3b9-0023591e2e58") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.hd7JsLesT5
+ rm /tmp/tmp.heus4y31YE /tmp/tmp.hd7JsLesT5
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JsvFAKNace
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gRuhkJGKIh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JsvFAKNace
++ cat /tmp/tmp.gRuhkJGKIh
++ rm /tmp/tmp.JsvFAKNace /tmp/tmp.gRuhkJGKIh
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ny0ETceYlH
++ mktemp
+ local LAST_ERR=/tmp/tmp.79XUlgeihm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ny0ETceYlH
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("7be272e4-b5c3-45b1-bed7-6478be5127ea") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.79XUlgeihm
+ rm /tmp/tmp.Ny0ETceYlH /tmp/tmp.79XUlgeihm
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NeiuvkeuKX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y3EMsAWIBG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NeiuvkeuKX
++ cat /tmp/tmp.y3EMsAWIBG
++ rm /tmp/tmp.NeiuvkeuKX /tmp/tmp.y3EMsAWIBG
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lYbsDSMNRm
++ mktemp
+ local LAST_ERR=/tmp/tmp.1XGdvSe5OE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lYbsDSMNRm
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("9a54d67c-7218-4a68-843f-d0e8bfe7cb5e") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.1XGdvSe5OE
+ rm /tmp/tmp.lYbsDSMNRm /tmp/tmp.1XGdvSe5OE
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GsmEXEqXKh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KbAQffHjEw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GsmEXEqXKh
++ cat /tmp/tmp.KbAQffHjEw
++ rm /tmp/tmp.GsmEXEqXKh /tmp/tmp.KbAQffHjEw
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.UWuChRRWE0
++ mktemp
+ local LAST_ERR=/tmp/tmp.PhHcpz1NvR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UWuChRRWE0
+ cat /tmp/tmp.PhHcpz1NvR
+ rm /tmp/tmp.UWuChRRWE0 /tmp/tmp.PhHcpz1NvR
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.94rJ6ctuxD/find-2nd
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gnl8CcAVky
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xhY0Rzffs1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gnl8CcAVky
++ cat /tmp/tmp.xhY0Rzffs1
++ rm /tmp/tmp.gnl8CcAVky /tmp/tmp.xhY0Rzffs1
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yFYdJY9vMC
++ mktemp
+ local LAST_ERR=/tmp/tmp.IJkEpuc7ws
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yFYdJY9vMC
+ cat /tmp/tmp.IJkEpuc7ws
+ rm /tmp/tmp.yFYdJY9vMC /tmp/tmp.IJkEpuc7ws
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.94rJ6ctuxD/find-2nd1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pFH1bOCjOq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Qkui3NpPCk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pFH1bOCjOq
++ cat /tmp/tmp.Qkui3NpPCk
++ rm /tmp/tmp.pFH1bOCjOq /tmp/tmp.Qkui3NpPCk
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.uWz3tByLPP
++ mktemp
+ local LAST_ERR=/tmp/tmp.4odNwJNXZ1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uWz3tByLPP
+ cat /tmp/tmp.4odNwJNXZ1
+ rm /tmp/tmp.uWz3tByLPP /tmp/tmp.4odNwJNXZ1
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.94rJ6ctuxD/find-2nd2
+ run_restore backup-aws-s3
+ local backup_name=backup-aws-s3
+ /usr/bin/sed -e 's/name:/name: restore-backup-aws-s3/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/restore.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/backupName:/backupName: backup-aws-s3/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.7B5UvJ7KFl
++ mktemp
+ local LAST_ERR=/tmp/tmp.nQwFuqYgBs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7B5UvJ7KFl
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
+ cat /tmp/tmp.nQwFuqYgBs
+ rm /tmp/tmp.7B5UvJ7KFl /tmp/tmp.nQwFuqYgBs
+ return 0
+ wait_restore backup-aws-s3 some-name
+ local backup_name=backup-aws-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3 object to be createdOK
Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" .OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c9qk27oxYJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xpy5tGYPgO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.c9qk27oxYJ
++ cat /tmp/tmp.xpy5tGYPgO
++ rm /tmp/tmp.c9qk27oxYJ /tmp/tmp.xpy5tGYPgO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pRbkVkQejv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Mr2cwwkRwu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pRbkVkQejv
++ cat /tmp/tmp.Mr2cwwkRwu
++ rm /tmp/tmp.pRbkVkQejv /tmp/tmp.Mr2cwwkRwu
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6eaH3uQJuO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bAZVEeCYr3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6eaH3uQJuO
++ cat /tmp/tmp.bAZVEeCYr3
++ rm /tmp/tmp.6eaH3uQJuO /tmp/tmp.bAZVEeCYr3
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.60GXh7nJBN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mfb4hpE7eW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.60GXh7nJBN
++ cat /tmp/tmp.mfb4hpE7eW
++ rm /tmp/tmp.60GXh7nJBN /tmp/tmp.mfb4hpE7eW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EGvGVNEJEw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2N7BGsi7yO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EGvGVNEJEw
++ cat /tmp/tmp.2N7BGsi7yO
++ rm /tmp/tmp.EGvGVNEJEw /tmp/tmp.2N7BGsi7yO
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q149Jl8Klo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KT7DNAks9o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.q149Jl8Klo
++ cat /tmp/tmp.KT7DNAks9o
++ rm /tmp/tmp.q149Jl8Klo /tmp/tmp.KT7DNAks9o
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.wQ1bzlnpi8
++ mktemp
+ local LAST_ERR=/tmp/tmp.g7j1dIU3Oz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wQ1bzlnpi8
+ cat /tmp/tmp.g7j1dIU3Oz
+ rm /tmp/tmp.wQ1bzlnpi8 /tmp/tmp.g7j1dIU3Oz
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.94rJ6ctuxD/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aJkbmnh7lu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PiThutR7HQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aJkbmnh7lu
++ cat /tmp/tmp.PiThutR7HQ
++ rm /tmp/tmp.aJkbmnh7lu /tmp/tmp.PiThutR7HQ
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Udn52jpNZd
++ mktemp
+ local LAST_ERR=/tmp/tmp.jgMraVyTEp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Udn52jpNZd
+ cat /tmp/tmp.jgMraVyTEp
+ rm /tmp/tmp.Udn52jpNZd /tmp/tmp.jgMraVyTEp
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.94rJ6ctuxD/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vsjiX8AouY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Cu5ee1dH7y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vsjiX8AouY
++ cat /tmp/tmp.Cu5ee1dH7y
++ rm /tmp/tmp.vsjiX8AouY /tmp/tmp.Cu5ee1dH7y
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DuubzmJ6JP
++ mktemp
+ local LAST_ERR=/tmp/tmp.8CSLjqKnBZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DuubzmJ6JP
+ cat /tmp/tmp.8CSLjqKnBZ
+ rm /tmp/tmp.DuubzmJ6JP /tmp/tmp.8CSLjqKnBZ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.94rJ6ctuxD/find2
+ desc 'check backup and restore -- gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs
++ local backup_name=backup-gcp-cs
++ kubectl_bin get psmdb-backup backup-gcp-cs -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|azure://||'
++ sed 's|s3://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0wxWpks6PA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.D9shhPqC0o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0wxWpks6PA
++ cat /tmp/tmp.D9shhPqC0o
++ rm /tmp/tmp.0wxWpks6PA /tmp/tmp.D9shhPqC0o
++ return 0
+ backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:01Z
+ gunzip
+ curl -s https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:01Z/rs0/myApp.test.gz
+ gunzip
+ curl -s https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:01Z/rs1/myApp1.test.gz
+ gunzip
+ curl -s https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:01Z/rs2/myApp2.test.gz
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uQowNg8y58
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fBw8KkeEpA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uQowNg8y58
++ cat /tmp/tmp.fBw8KkeEpA
++ rm /tmp/tmp.uQowNg8y58 /tmp/tmp.fBw8KkeEpA
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Zr2IvFALBg
++ mktemp
+ local LAST_ERR=/tmp/tmp.p3izGwR9Zx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Zr2IvFALBg
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("abc874dd-f9df-4eb4-8e51-bb0f08488cc7") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.p3izGwR9Zx
+ rm /tmp/tmp.Zr2IvFALBg /tmp/tmp.p3izGwR9Zx
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ldP0X0awii
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5MDFwflKtf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ldP0X0awii
++ cat /tmp/tmp.5MDFwflKtf
++ rm /tmp/tmp.ldP0X0awii /tmp/tmp.5MDFwflKtf
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WZ8Jf7lGv0
++ mktemp
+ local LAST_ERR=/tmp/tmp.PrECa4ryvQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WZ8Jf7lGv0
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("72d3c654-a63e-4a95-b337-7cf030206139") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.PrECa4ryvQ
+ rm /tmp/tmp.WZ8Jf7lGv0 /tmp/tmp.PrECa4ryvQ
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lDp10W3iUc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ot38w5iLUq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lDp10W3iUc
++ cat /tmp/tmp.Ot38w5iLUq
++ rm /tmp/tmp.lDp10W3iUc /tmp/tmp.Ot38w5iLUq
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OT4Iv98Vvj
++ mktemp
+ local LAST_ERR=/tmp/tmp.IoF4q9kN6p
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OT4Iv98Vvj
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("85597adc-cb05-418d-857b-be999f67d600") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.IoF4q9kN6p
+ rm /tmp/tmp.OT4Iv98Vvj /tmp/tmp.IoF4q9kN6p
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WkcsTeUlNJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.USqvgLMjQV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WkcsTeUlNJ
++ cat /tmp/tmp.USqvgLMjQV
++ rm /tmp/tmp.WkcsTeUlNJ /tmp/tmp.USqvgLMjQV
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mtx4p6EDIp
++ mktemp
+ local LAST_ERR=/tmp/tmp.P27uD4pxd5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mtx4p6EDIp
+ cat /tmp/tmp.P27uD4pxd5
+ rm /tmp/tmp.mtx4p6EDIp /tmp/tmp.P27uD4pxd5
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.94rJ6ctuxD/find-2nd
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S2X8t5DbJf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RR70tpjCeq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.S2X8t5DbJf
++ cat /tmp/tmp.RR70tpjCeq
++ rm /tmp/tmp.S2X8t5DbJf /tmp/tmp.RR70tpjCeq
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pjCuifROhe
++ mktemp
+ local LAST_ERR=/tmp/tmp.TFwHHi1czs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pjCuifROhe
+ cat /tmp/tmp.TFwHHi1czs
+ rm /tmp/tmp.pjCuifROhe /tmp/tmp.TFwHHi1czs
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.94rJ6ctuxD/find-2nd1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.njbTBT3win
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Y5HY3YD98o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.njbTBT3win
++ cat /tmp/tmp.Y5HY3YD98o
++ rm /tmp/tmp.njbTBT3win /tmp/tmp.Y5HY3YD98o
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.t4HY7lv7Y5
++ mktemp
+ local LAST_ERR=/tmp/tmp.0TMoMw0TsC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t4HY7lv7Y5
+ cat /tmp/tmp.0TMoMw0TsC
+ rm /tmp/tmp.t4HY7lv7Y5 /tmp/tmp.0TMoMw0TsC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.94rJ6ctuxD/find-2nd2
+ run_restore backup-gcp-cs
+ local backup_name=backup-gcp-cs
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-gcp-cs/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-gcp-cs/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.2iffhKCU5V
++ mktemp
+ local LAST_ERR=/tmp/tmp.7cXWOIdSLz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2iffhKCU5V
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs created
+ cat /tmp/tmp.7cXWOIdSLz
+ rm /tmp/tmp.2iffhKCU5V /tmp/tmp.7cXWOIdSLz
+ return 0
+ wait_restore backup-gcp-cs some-name
+ local backup_name=backup-gcp-cs
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs object to be createdOK
Waiting psmdb-restore/restore-backup-gcp-cs to reach state "ready" .OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZI7Ecyrcuc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9qiXYAHAtL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZI7Ecyrcuc
++ cat /tmp/tmp.9qiXYAHAtL
++ rm /tmp/tmp.ZI7Ecyrcuc /tmp/tmp.9qiXYAHAtL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.A9IbZM2CGC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yP2jCYA3rw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.A9IbZM2CGC
++ cat /tmp/tmp.yP2jCYA3rw
++ rm /tmp/tmp.A9IbZM2CGC /tmp/tmp.yP2jCYA3rw
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u2yZy7dVCl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NBm4asDqsg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.u2yZy7dVCl
++ cat /tmp/tmp.NBm4asDqsg
++ rm /tmp/tmp.u2yZy7dVCl /tmp/tmp.NBm4asDqsg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oP457IFUvb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Y3SHLlEr2J
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oP457IFUvb
++ cat /tmp/tmp.Y3SHLlEr2J
++ rm /tmp/tmp.oP457IFUvb /tmp/tmp.Y3SHLlEr2J
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9lgPVOqJzt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tSsoRsf9X8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9lgPVOqJzt
++ cat /tmp/tmp.tSsoRsf9X8
++ rm /tmp/tmp.9lgPVOqJzt /tmp/tmp.tSsoRsf9X8
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.axEm5dGEtP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.V1FLW0bUmm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.axEm5dGEtP
++ cat /tmp/tmp.V1FLW0bUmm
++ rm /tmp/tmp.axEm5dGEtP /tmp/tmp.V1FLW0bUmm
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qxY9Fyf8gq
++ mktemp
+ local LAST_ERR=/tmp/tmp.NVqCRGgNNC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qxY9Fyf8gq
+ cat /tmp/tmp.NVqCRGgNNC
+ rm /tmp/tmp.qxY9Fyf8gq /tmp/tmp.NVqCRGgNNC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.94rJ6ctuxD/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ybNsmLNMRb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vRqka8CLPm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ybNsmLNMRb
++ cat /tmp/tmp.vRqka8CLPm
++ rm /tmp/tmp.ybNsmLNMRb /tmp/tmp.vRqka8CLPm
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.iXp5Jvdi25
++ mktemp
+ local LAST_ERR=/tmp/tmp.CHhvH6neup
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iXp5Jvdi25
+ cat /tmp/tmp.CHhvH6neup
+ rm /tmp/tmp.iXp5Jvdi25 /tmp/tmp.CHhvH6neup
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.94rJ6ctuxD/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UcZC17NPvD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.api5682zU9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UcZC17NPvD
++ cat /tmp/tmp.api5682zU9
++ rm /tmp/tmp.UcZC17NPvD /tmp/tmp.api5682zU9
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0r9gZfFgum
++ mktemp
+ local LAST_ERR=/tmp/tmp.EYra2e1451
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0r9gZfFgum
+ cat /tmp/tmp.EYra2e1451
+ rm /tmp/tmp.0r9gZfFgum /tmp/tmp.EYra2e1451
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.94rJ6ctuxD/find2
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.9zqDSiBNqQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H3cnTHYUxF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9zqDSiBNqQ
++ cat /tmp/tmp.H3cnTHYUxF
++ rm /tmp/tmp.9zqDSiBNqQ /tmp/tmp.H3cnTHYUxF
++ return 0
+ backup_dest_azure=operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:34Z
+ curl -s https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:34Z/rs0/myApp.test.gz
+ gunzip
+ gunzip
+ curl -s https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:34Z/rs1/myApp1.test.gz
+ curl -s https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:34Z/rs2/myApp2.test.gz
+ gunzip
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wZIeDRmxQb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fwlooU4nNc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wZIeDRmxQb
++ cat /tmp/tmp.fwlooU4nNc
++ rm /tmp/tmp.wZIeDRmxQb /tmp/tmp.fwlooU4nNc
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.XPlnB72Cm2
++ mktemp
+ local LAST_ERR=/tmp/tmp.9ZeV9wayWe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XPlnB72Cm2
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("8123eea6-8e6f-49ea-b2af-0276d1cb4a2b") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.9ZeV9wayWe
+ rm /tmp/tmp.XPlnB72Cm2 /tmp/tmp.9ZeV9wayWe
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0EsDbiiJWn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1e1yGKm5fV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0EsDbiiJWn
++ cat /tmp/tmp.1e1yGKm5fV
++ rm /tmp/tmp.0EsDbiiJWn /tmp/tmp.1e1yGKm5fV
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.RDByT0AB81
++ mktemp
+ local LAST_ERR=/tmp/tmp.g2uMiR69Zv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RDByT0AB81
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("c9746a0c-8f80-45ca-a7ee-e7d62d221c83") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.g2uMiR69Zv
+ rm /tmp/tmp.RDByT0AB81 /tmp/tmp.g2uMiR69Zv
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iowKjxm8AL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XuZ4Qa09VV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iowKjxm8AL
++ cat /tmp/tmp.XuZ4Qa09VV
++ rm /tmp/tmp.iowKjxm8AL /tmp/tmp.XuZ4Qa09VV
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.aop6DsFdrP
++ mktemp
+ local LAST_ERR=/tmp/tmp.vU1AK5XJbO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aop6DsFdrP
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e708a0a4-3c0f-41f0-906c-e9cebe53f948") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.vU1AK5XJbO
+ rm /tmp/tmp.aop6DsFdrP /tmp/tmp.vU1AK5XJbO
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wKQPDu3TKw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m6tLSvQbId
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wKQPDu3TKw
++ cat /tmp/tmp.m6tLSvQbId
++ rm /tmp/tmp.wKQPDu3TKw /tmp/tmp.m6tLSvQbId
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.obIM8xYJKM
++ mktemp
+ local LAST_ERR=/tmp/tmp.XAuotFfHc6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.obIM8xYJKM
+ cat /tmp/tmp.XAuotFfHc6
+ rm /tmp/tmp.obIM8xYJKM /tmp/tmp.XAuotFfHc6
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.94rJ6ctuxD/find-2nd
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tIOZIE3XKl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.e6NzD7kFL9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tIOZIE3XKl
++ cat /tmp/tmp.e6NzD7kFL9
++ rm /tmp/tmp.tIOZIE3XKl /tmp/tmp.e6NzD7kFL9
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.IQU2x2cwjg
++ mktemp
+ local LAST_ERR=/tmp/tmp.KXfx6d5dgd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IQU2x2cwjg
+ cat /tmp/tmp.KXfx6d5dgd
+ rm /tmp/tmp.IQU2x2cwjg /tmp/tmp.KXfx6d5dgd
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.94rJ6ctuxD/find-2nd1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vS0xlbeRRg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6mr87vVw7K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vS0xlbeRRg
++ cat /tmp/tmp.6mr87vVw7K
++ rm /tmp/tmp.vS0xlbeRRg /tmp/tmp.6mr87vVw7K
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.o9Y4TZbBDM
++ mktemp
+ local LAST_ERR=/tmp/tmp.lohbDLUBCx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.o9Y4TZbBDM
+ cat /tmp/tmp.lohbDLUBCx
+ rm /tmp/tmp.o9Y4TZbBDM /tmp/tmp.lohbDLUBCx
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.94rJ6ctuxD/find-2nd2
+ run_restore backup-azure-blob
+ local backup_name=backup-azure-blob
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/bin/sed -e 's/backupName:/backupName: backup-azure-blob/'
+ /usr/bin/sed -e 's/name:/name: restore-backup-azure-blob/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.PbTqsi1aAp
++ mktemp
+ local LAST_ERR=/tmp/tmp.MZOAatm8zs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PbTqsi1aAp
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
+ cat /tmp/tmp.MZOAatm8zs
+ rm /tmp/tmp.PbTqsi1aAp /tmp/tmp.MZOAatm8zs
+ return 0
+ wait_restore backup-azure-blob some-name
+ local backup_name=backup-azure-blob
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob object to be createdOK
Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" .OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WNxCVksGqg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UxsvjCt7tD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WNxCVksGqg
++ cat /tmp/tmp.UxsvjCt7tD
++ rm /tmp/tmp.WNxCVksGqg /tmp/tmp.UxsvjCt7tD
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZKYg0xb78Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZlLw84vvMt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZKYg0xb78Q
++ cat /tmp/tmp.ZlLw84vvMt
++ rm /tmp/tmp.ZKYg0xb78Q /tmp/tmp.ZlLw84vvMt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FtAJXrIF40
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GML3RinypP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FtAJXrIF40
++ cat /tmp/tmp.GML3RinypP
++ rm /tmp/tmp.FtAJXrIF40 /tmp/tmp.GML3RinypP
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6snjScykBe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SVTt7LUcXl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6snjScykBe
++ cat /tmp/tmp.SVTt7LUcXl
++ rm /tmp/tmp.6snjScykBe /tmp/tmp.SVTt7LUcXl
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zcjUY6ghIl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iXbHZUlfdi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zcjUY6ghIl
++ cat /tmp/tmp.iXbHZUlfdi
++ rm /tmp/tmp.zcjUY6ghIl /tmp/tmp.iXbHZUlfdi
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Je9eeHhrfn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZlcvELcVlz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Je9eeHhrfn
++ cat /tmp/tmp.ZlcvELcVlz
++ rm /tmp/tmp.Je9eeHhrfn /tmp/tmp.ZlcvELcVlz
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Xm874qwgFt
++ mktemp
+ local LAST_ERR=/tmp/tmp.hkJmgjQSbr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Xm874qwgFt
+ cat /tmp/tmp.hkJmgjQSbr
+ rm /tmp/tmp.Xm874qwgFt /tmp/tmp.hkJmgjQSbr
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.94rJ6ctuxD/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6lKfCaJHSe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sDaA47liLk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6lKfCaJHSe
++ cat /tmp/tmp.sDaA47liLk
++ rm /tmp/tmp.6lKfCaJHSe /tmp/tmp.sDaA47liLk
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jcnNaa3I2X
++ mktemp
+ local LAST_ERR=/tmp/tmp.4MbmZwibUD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jcnNaa3I2X
+ cat /tmp/tmp.4MbmZwibUD
+ rm /tmp/tmp.jcnNaa3I2X /tmp/tmp.4MbmZwibUD
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.94rJ6ctuxD/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DbHjF2khB9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r1SWCxsNbZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DbHjF2khB9
++ cat /tmp/tmp.r1SWCxsNbZ
++ rm /tmp/tmp.DbHjF2khB9 /tmp/tmp.r1SWCxsNbZ
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.6dIlB4GHOD
++ mktemp
+ local LAST_ERR=/tmp/tmp.1O1YiM0xF6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6dIlB4GHOD
+ cat /tmp/tmp.1O1YiM0xF6
+ rm /tmp/tmp.6dIlB4GHOD /tmp/tmp.1O1YiM0xF6
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.94rJ6ctuxD/find2
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

+ check_backup_in_storage backup-minio minio rs0 myApp.test.gz
+ local backup=backup-minio
+ local storage_type=minio
+ local replset=rs0
+ local file=myApp.test.gz
+ local endpoint
+ case ${storage_type} in
+ endpoint=minio-service
++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ /usr/bin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed 's|s3://||'
+++ mktemp
++ sed -e 's/.json$//'
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.c4ccISBHku
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xx4aAlwk0V
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.c4ccISBHku
++ cat /tmp/tmp.xx4aAlwk0V
++ rm /tmp/tmp.c4ccISBHku /tmp/tmp.xx4aAlwk0V
++ return 0
+ backup_dest=operator-testing/2025-08-06T21:04:55Z
+ [[ minio == \m\i\n\i\o ]]
+ grep myApp.test.gz
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-08-06T21:04:55Z/rs0/myApp.test.gz
++ mktemp
+ local LAST_OUT=/tmp/tmp.BGBpnpcSi3
++ mktemp
+ local LAST_ERR=/tmp/tmp.Je1fKF0DIM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-08-06T21:04:55Z/rs0/myApp.test.gz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BGBpnpcSi3
+ cat /tmp/tmp.Je1fKF0DIM
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.BGBpnpcSi3 /tmp/tmp.Je1fKF0DIM
+ return 0
2025-08-06 21:04:59         56 myApp.test.gz
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.T4TIL7LsdB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lffunGdCJS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.T4TIL7LsdB
++ cat /tmp/tmp.lffunGdCJS
++ rm /tmp/tmp.T4TIL7LsdB /tmp/tmp.lffunGdCJS
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ebyFKMKOV8
++ mktemp
+ local LAST_ERR=/tmp/tmp.9zmdyggsL6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ebyFKMKOV8
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("0df9e258-bea0-411a-b11f-ef56163dbec9") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.9zmdyggsL6
+ rm /tmp/tmp.ebyFKMKOV8 /tmp/tmp.9zmdyggsL6
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7d53FCt9WJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zyXsdfC75a
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7d53FCt9WJ
++ cat /tmp/tmp.zyXsdfC75a
++ rm /tmp/tmp.7d53FCt9WJ /tmp/tmp.zyXsdfC75a
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WBov6z64PS
++ mktemp
+ local LAST_ERR=/tmp/tmp.LM2dOkVM1d
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WBov6z64PS
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("038ffb90-4ba2-4e49-92fa-c8c16ee36cd7") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.LM2dOkVM1d
+ rm /tmp/tmp.WBov6z64PS /tmp/tmp.LM2dOkVM1d
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P0YeFQ1BVe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.alYe1oGupQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.P0YeFQ1BVe
++ cat /tmp/tmp.alYe1oGupQ
++ rm /tmp/tmp.P0YeFQ1BVe /tmp/tmp.alYe1oGupQ
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.eK1cZ9MeL1
++ mktemp
+ local LAST_ERR=/tmp/tmp.7JvEKUQgGK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eK1cZ9MeL1
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("c360e722-c64d-461e-825d-fdc6a0299fff") }
Percona Server for MongoDB server version: v7.0.22-12
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.7JvEKUQgGK
+ rm /tmp/tmp.eK1cZ9MeL1 /tmp/tmp.7JvEKUQgGK
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VEbr6rEtA0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ff3q2Y7fNC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VEbr6rEtA0
++ cat /tmp/tmp.ff3q2Y7fNC
++ rm /tmp/tmp.VEbr6rEtA0 /tmp/tmp.ff3q2Y7fNC
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.B8t1hoVrJ0
++ mktemp
+ local LAST_ERR=/tmp/tmp.u7SB4zx0J9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.B8t1hoVrJ0
+ cat /tmp/tmp.u7SB4zx0J9
+ rm /tmp/tmp.B8t1hoVrJ0 /tmp/tmp.u7SB4zx0J9
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.94rJ6ctuxD/find-2nd
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TqpWA2kKZG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YahWLBC9JC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TqpWA2kKZG
++ cat /tmp/tmp.YahWLBC9JC
++ rm /tmp/tmp.TqpWA2kKZG /tmp/tmp.YahWLBC9JC
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.IgscR2w5xx
++ mktemp
+ local LAST_ERR=/tmp/tmp.D2JV7fDZoY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IgscR2w5xx
+ cat /tmp/tmp.D2JV7fDZoY
+ rm /tmp/tmp.IgscR2w5xx /tmp/tmp.D2JV7fDZoY
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.94rJ6ctuxD/find-2nd1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hq5EfbP3uL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f6oGd0FFU5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hq5EfbP3uL
++ cat /tmp/tmp.f6oGd0FFU5
++ rm /tmp/tmp.hq5EfbP3uL /tmp/tmp.f6oGd0FFU5
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.C45NVRonT3
++ mktemp
+ local LAST_ERR=/tmp/tmp.8daSdiXSBp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C45NVRonT3
+ cat /tmp/tmp.8daSdiXSBp
+ rm /tmp/tmp.C45NVRonT3 /tmp/tmp.8daSdiXSBp
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.94rJ6ctuxD/find-2nd2
+ run_restore backup-minio
+ local backup_name=backup-minio
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.eYbEaoefwR
++ mktemp
+ local LAST_ERR=/tmp/tmp.NlASgieMB3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eYbEaoefwR
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
+ cat /tmp/tmp.NlASgieMB3
+ rm /tmp/tmp.eYbEaoefwR /tmp/tmp.NlASgieMB3
+ return 0
+ wait_restore backup-minio some-name
+ local backup_name=backup-minio
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio object to be createdOK
Waiting psmdb-restore/restore-backup-minio to reach state "ready" .OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5hX1IbK11G
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9albPNR22T
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5hX1IbK11G
++ cat /tmp/tmp.9albPNR22T
++ rm /tmp/tmp.5hX1IbK11G /tmp/tmp.9albPNR22T
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JXmVghFqs6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ct7ikXLvII
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JXmVghFqs6
++ cat /tmp/tmp.ct7ikXLvII
++ rm /tmp/tmp.JXmVghFqs6 /tmp/tmp.ct7ikXLvII
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y954R4AToh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bu3HZXVDEr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y954R4AToh
++ cat /tmp/tmp.bu3HZXVDEr
++ rm /tmp/tmp.y954R4AToh /tmp/tmp.bu3HZXVDEr
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YLJFjVMXOk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qdLQh22eN7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YLJFjVMXOk
++ cat /tmp/tmp.qdLQh22eN7
++ rm /tmp/tmp.YLJFjVMXOk /tmp/tmp.qdLQh22eN7
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wXzAhL3mYD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dm2PC0uWeT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wXzAhL3mYD
++ cat /tmp/tmp.dm2PC0uWeT
++ rm /tmp/tmp.wXzAhL3mYD /tmp/tmp.dm2PC0uWeT
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ check_data
+ local postfix=
++ seq 0 2
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nEdLxJvevI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sY4K9ZXs3x
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nEdLxJvevI
++ cat /tmp/tmp.sY4K9ZXs3x
++ rm /tmp/tmp.nEdLxJvevI /tmp/tmp.sY4K9ZXs3x
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OcKO8uPMZC
++ mktemp
+ local LAST_ERR=/tmp/tmp.y3I7d2fwYH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OcKO8uPMZC
+ cat /tmp/tmp.y3I7d2fwYH
+ rm /tmp/tmp.OcKO8uPMZC /tmp/tmp.y3I7d2fwYH
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.94rJ6ctuxD/find
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local port=27019
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lSSYt4gMFa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Xqancr7Gbb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lSSYt4gMFa
++ cat /tmp/tmp.Xqancr7Gbb
++ rm /tmp/tmp.lSSYt4gMFa /tmp/tmp.Xqancr7Gbb
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.NghzTTRYyM
++ mktemp
+ local LAST_ERR=/tmp/tmp.DCjzEIjwzR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NghzTTRYyM
+ cat /tmp/tmp.DCjzEIjwzR
+ rm /tmp/tmp.NghzTTRYyM /tmp/tmp.DCjzEIjwzR
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.94rJ6ctuxD/find1
+ for i in '$(seq 0 2)'
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-19519 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-19519 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-19519
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k3G0P1gAtE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xvCR5b0Chi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.k3G0P1gAtE
++ cat /tmp/tmp.xvCR5b0Chi
++ rm /tmp/tmp.k3G0P1gAtE /tmp/tmp.xvCR5b0Chi
++ return 0
+ local client_container=psmdb-client-66f577db5f-tsrr9
+ kubectl_bin exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vNhzhgYIc5
++ mktemp
+ local LAST_ERR=/tmp/tmp.rxpVAlXekS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-tsrr9 -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-19519.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vNhzhgYIc5
+ cat /tmp/tmp.rxpVAlXekS
+ rm /tmp/tmp.vNhzhgYIc5 /tmp/tmp.rxpVAlXekS
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.94rJ6ctuxD/find2
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.JEhFMYAXUb
++ mktemp
+ local LAST_ERR=/tmp/tmp.VmI9oSqkc9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JEhFMYAXUb
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted
+ cat /tmp/tmp.VmI9oSqkc9
+ rm /tmp/tmp.JEhFMYAXUb /tmp/tmp.VmI9oSqkc9
+ return 0
++ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ grep -c _
++ cat
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Nd1TjNv6C5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IweNnMWy9j
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Nd1TjNv6C5
++ cat /tmp/tmp.IweNnMWy9j
++ rm /tmp/tmp.Nd1TjNv6C5 /tmp/tmp.IweNnMWy9j
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:05:27Z aws-s3
+ path=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:05:27Z
+ storage_name=aws-s3
+ retry=0
++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:05:27Z
+ [[ 403 -eq 403 ]]
+ check_backup_deletion https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:01Z gcp-cs
+ path=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:01Z
+ storage_name=gcp-cs
+ retry=0
++ curl -sw '%{http_code}' -o /dev/null https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:01Z
+ [[ 404 -eq 403 ]]
++ curl -sw '%{http_code}' -o /dev/null https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:01Z
+ [[ 404 -eq 404 ]]
+ check_backup_deletion https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:34Z azure-blob
+ path=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:34Z
+ storage_name=azure-blob
+ retry=0
++ curl -sw '%{http_code}' -o /dev/null https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:34Z
+ [[ 404 -eq 403 ]]
++ curl -sw '%{http_code}' -o /dev/null https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-08-06T21:06:34Z
+ [[ 404 -eq 404 ]]
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/container-rc.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.3SU6Y7RAWx
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZwdIzSzjD1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/container-rc.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3SU6Y7RAWx
runtimeclass.node.k8s.io "container-rc" deleted
+ cat /tmp/tmp.ZwdIzSzjD1
+ rm /tmp/tmp.3SU6Y7RAWx /tmp/tmp.ZwdIzSzjD1
+ return 0
+ destroy demand-backup-sharded-19519
+ local namespace=demand-backup-sharded-19519
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u8mWheGMWH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v6dob9V1LA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.u8mWheGMWH
++ cat /tmp/tmp.v6dob9V1LA
No resources found in demand-backup-sharded-19519 namespace.
++ rm /tmp/tmp.u8mWheGMWH /tmp/tmp.v6dob9V1LA
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.0SMgL6iR5h
++ mktemp
+ local LAST_ERR=/tmp/tmp.OPy8Ol01sL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0SMgL6iR5h
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.OPy8Ol01sL
+ rm /tmp/tmp.0SMgL6iR5h /tmp/tmp.OPy8Ol01sL
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.9mmabCmmq4
++ mktemp
+ local LAST_ERR=/tmp/tmp.FUn8uTxc8k
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9mmabCmmq4
+ cat /tmp/tmp.FUn8uTxc8k
+ rm /tmp/tmp.9mmabCmmq4 /tmp/tmp.FUn8uTxc8k
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.TZ6GQEFrM3
++ mktemp
+ local LAST_ERR=/tmp/tmp.i6o7Uw9cwD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TZ6GQEFrM3
+ cat /tmp/tmp.i6o7Uw9cwD
+ rm /tmp/tmp.TZ6GQEFrM3 /tmp/tmp.i6o7Uw9cwD
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Kshf0oQOg0
++ mktemp
+ local LAST_ERR=/tmp/tmp.LkSaj7ces2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Kshf0oQOg0
+ cat /tmp/tmp.LkSaj7ces2
+ rm /tmp/tmp.Kshf0oQOg0 /tmp/tmp.LkSaj7ces2
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.hBNUYz0meN
++ mktemp
+ local LAST_ERR=/tmp/tmp.1eALIlyqea
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hBNUYz0meN
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.1eALIlyqea
+ rm /tmp/tmp.hBNUYz0meN /tmp/tmp.1eALIlyqea
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.od5PabCxHs
++ mktemp
+ local LAST_ERR=/tmp/tmp.DdBJMLxavh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.od5PabCxHs
+ cat /tmp/tmp.DdBJMLxavh
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.od5PabCxHs
+ cat /tmp/tmp.DdBJMLxavh
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.od5PabCxHs
+ cat /tmp/tmp.DdBJMLxavh
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.od5PabCxHs
+ cat /tmp/tmp.DdBJMLxavh
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.od5PabCxHs /tmp/tmp.DdBJMLxavh
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ rm -rf /tmp/tmp.94rJ6ctuxD
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-sharded-19519
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.FiIbu9CtEU
+ local LAST_OUT=/tmp/tmp.BwHPnodkVK
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.shYzc5BJkp
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.mE5HI5OEEX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-sharded-19519